{"meta":{"date":"2026-08-24","timezone":"UTC","generated_at":"2026-08-25T22:04:42Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":103,"failed_runs":90,"runs_with_occurrences":90,"runs_with_failure_pattern_matches":83,"runs_with_unmatched_occurrences":7,"runs_without_occurrence_rows":13,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092038436008497152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5958,"pr_state":"open","pr_sha":"8f8a842137b7cc717d60c037df240a3633bc5e18","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-24T23:56:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6554,"pr_state":"open","pr_sha":"4e67bc77de1a80644494c4c53d6416cbb8167300","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T23:41:48Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2dd3865f6a7180a89c50cee7f4b354dcd33a59f08c29fcb04933e65b6160a360","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","failed_at":"alert","signature_id":"2f1d9b2b8857e75d809a23da90c2e32bd0a70345c25f6862fc7292ac5c796db3","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T00:35:25Z\n Ended: 2026-08-25T00:56:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1843200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd9551d19b6bblvufm3l3a421htavmi-a4k0z5e8x0e3t0f\", pod=\"router-597d99bc98-tlmg9\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd9551d19b6bblvufm3l3a421htavmi-a4k0z5e8x0e3t0f/router-597d99bc98-tlmg9 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"b658eb15aef6d1c79d1566c934a0193b744d31fde65af90a8a6dd4f6c56dc8f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","failed_at":"e2e","signature_id":"93cd087eb0fc0ac2b0c79ba3bc6c6181f71cba9db1ac3eaefd4c1d8152fbb877","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b66310\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[00:48:59.861]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:48:54.9335446Z);\\nlet _endTime = datetime(2026-08-25T00:48:54.9335452Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:48:59.8208131Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45\\\",\\r\\n \\\"activityId\\\": \\\"3f29cef9-55f3-4e39-a44c-dc2745aeb329\\\",\\r\\n \\\"subActivityId\\\": \\\"9dc2e9b4-43ec-49f2-b454-4f12f219c5db\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9b9edaeb-e277-470d-a49a-ba5e0f233409\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45 ARID=3f29cef9-55f3-4e39-a44c-dc2745aeb329 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9b9edaeb-e277-470d-a49a-ba5e0f233409 \\u003e DN.FE.ExecuteQuery/9dc2e9b4-43ec-49f2-b454-4f12f219c5db)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:48:59.984]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:48:54.9335446Z);\\nlet _endTime = datetime(2026-08-25T00:48:54.9335452Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:48:59.8208131Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45\\\",\\r\\n \\\"activityId\\\": \\\"3f29cef9-55f3-4e39-a44c-dc2745aeb329\\\",\\r\\n \\\"subActivityId\\\": \\\"9dc2e9b4-43ec-49f2-b454-4f12f219c5db\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9b9edaeb-e277-470d-a49a-ba5e0f233409\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45 ARID=3f29cef9-55f3-4e39-a44c-dc2745aeb329 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9b9edaeb-e277-470d-a49a-ba5e0f233409 \\u003e DN.FE.ExecuteQuery/9dc2e9b4-43ec-49f2-b454-4f12f219c5db)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[00:49:04.412]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:00.0668173Z);\\nlet _endTime = datetime(2026-08-25T00:49:00.0668177Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:49:04.3919639Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef\\\",\\r\\n \\\"activityId\\\": \\\"590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea\\\",\\r\\n \\\"subActivityId\\\": \\\"f16f10a7-70e1-48c7-9e9d-209207cfc1e8\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"69739f9c-88ef-46ee-93e1-65af7b0a2416\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef ARID=590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/69739f9c-88ef-46ee-93e1-65af7b0a2416 \\u003e DN.FE.ExecuteQuery/f16f10a7-70e1-48c7-9e9d-209207cfc1e8)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:49:04.464]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:00.0668173Z);\\nlet _endTime = datetime(2026-08-25T00:49:00.0668177Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:49:04.3919639Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef\\\",\\r\\n \\\"activityId\\\": \\\"590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea\\\",\\r\\n \\\"subActivityId\\\": \\\"f16f10a7-70e1-48c7-9e9d-209207cfc1e8\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"69739f9c-88ef-46ee-93e1-65af7b0a2416\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef ARID=590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/69739f9c-88ef-46ee-93e1-65af7b0a2416 \\u003e DN.FE.ExecuteQuery/f16f10a7-70e1-48c7-9e9d-209207cfc1e8)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[00:49:12.903]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:08.6625258Z);\\nlet _endTime = datetime(2026-08-25T00:49:08.6625262Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:49:12.8815919Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a5c94555-e249-4511-a807-b8477933031f\\\",\\r\\n \\\"activityId\\\": \\\"3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c\\\",\\r\\n \\\"subActivityId\\\": \\\"af4bd91d-24d7-4768-a750-045fb74a6811\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a5c94555-e249-4511-a807-b8477933031f ARID=3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa \\u003e DN.FE.ExecuteQuery/af4bd91d-24d7-4768-a750-045fb74a6811)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:49:13.008]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:08.6625258Z);\\nlet _endTime = datetime(2026-08-25T00:49:08.6625262Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:49:12.8815919Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a5c94555-e249-4511-a807-b8477933031f\\\",\\r\\n \\\"activityId\\\": \\\"3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c\\\",\\r\\n \\\"subActivityId\\\": \\\"af4bd91d-24d7-4768-a750-045fb74a6811\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a5c94555-e249-4511-a807-b8477933031f ARID=3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa \\u003e DN.FE.ExecuteQuery/af4bd91d-24d7-4768-a750-045fb74a6811)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b66310\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[00:48:59.861]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:48:54.9335446Z);\\nlet _endTime = datetime(2026-08-25T00:48:54.9335452Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:48:59.8208131Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45\\\",\\r\\n \\\"activityId\\\": \\\"3f29cef9-55f3-4e39-a44c-dc2745aeb329\\\",\\r\\n \\\"subActivityId\\\": \\\"9dc2e9b4-43ec-49f2-b454-4f12f219c5db\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9b9edaeb-e277-470d-a49a-ba5e0f233409\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45 ARID=3f29cef9-55f3-4e39-a44c-dc2745aeb329 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9b9edaeb-e277-470d-a49a-ba5e0f233409 \\u003e DN.FE.ExecuteQuery/9dc2e9b4-43ec-49f2-b454-4f12f219c5db)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:48:59.984]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:48:54.9335446Z);\\nlet _endTime = datetime(2026-08-25T00:48:54.9335452Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:48:59.8208131Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45\\\",\\r\\n \\\"activityId\\\": \\\"3f29cef9-55f3-4e39-a44c-dc2745aeb329\\\",\\r\\n \\\"subActivityId\\\": \\\"9dc2e9b4-43ec-49f2-b454-4f12f219c5db\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9b9edaeb-e277-470d-a49a-ba5e0f233409\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45 ARID=3f29cef9-55f3-4e39-a44c-dc2745aeb329 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9b9edaeb-e277-470d-a49a-ba5e0f233409 \\u003e DN.FE.ExecuteQuery/9dc2e9b4-43ec-49f2-b454-4f12f219c5db)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[00:49:04.412]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:00.0668173Z);\\nlet _endTime = datetime(2026-08-25T00:49:00.0668177Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:49:04.3919639Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef\\\",\\r\\n \\\"activityId\\\": \\\"590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea\\\",\\r\\n \\\"subActivityId\\\": \\\"f16f10a7-70e1-48c7-9e9d-209207cfc1e8\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"69739f9c-88ef-46ee-93e1-65af7b0a2416\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef ARID=590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/69739f9c-88ef-46ee-93e1-65af7b0a2416 \\u003e DN.FE.ExecuteQuery/f16f10a7-70e1-48c7-9e9d-209207cfc1e8)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:49:04.464]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:00.0668173Z);\\nlet _endTime = datetime(2026-08-25T00:49:00.0668177Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:49:04.3919639Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef\\\",\\r\\n \\\"activityId\\\": \\\"590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea\\\",\\r\\n \\\"subActivityId\\\": \\\"f16f10a7-70e1-48c7-9e9d-209207cfc1e8\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"69739f9c-88ef-46ee-93e1-65af7b0a2416\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef ARID=590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/69739f9c-88ef-46ee-93e1-65af7b0a2416 \\u003e DN.FE.ExecuteQuery/f16f10a7-70e1-48c7-9e9d-209207cfc1e8)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[00:49:12.903]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:08.6625258Z);\\nlet _endTime = datetime(2026-08-25T00:49:08.6625262Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:49:12.8815919Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a5c94555-e249-4511-a807-b8477933031f\\\",\\r\\n \\\"activityId\\\": \\\"3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c\\\",\\r\\n \\\"subActivityId\\\": \\\"af4bd91d-24d7-4768-a750-045fb74a6811\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a5c94555-e249-4511-a807-b8477933031f ARID=3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa \\u003e DN.FE.ExecuteQuery/af4bd91d-24d7-4768-a750-045fb74a6811)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:49:13.008]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:08.6625258Z);\\nlet _endTime = datetime(2026-08-25T00:49:08.6625262Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:49:12.8815919Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a5c94555-e249-4511-a807-b8477933031f\\\",\\r\\n \\\"activityId\\\": \\\"3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c\\\",\\r\\n \\\"subActivityId\\\": \\\"af4bd91d-24d7-4768-a750-045fb74a6811\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a5c94555-e249-4511-a807-b8477933031f ARID=3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa \\u003e DN.FE.ExecuteQuery/af4bd91d-24d7-4768-a750-045fb74a6811)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"d254f8612cd03ec24a7fb9d82bdf3646858b06c03c45d7328456a9d814c3b26e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","failed_at":"e2e","signature_id":"54a530c9876fc008fec6ac6899f443f3e3f5628a64422efb1e6064e8186af6e1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-h74snzg5tpbh/cluster-candidate-5-0-gp9ztx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014a05c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-gp9ztx: failed waiting for cluster=\"cluster-candidate-5-0-gp9ztx\" in resourcegroup=\"rg-candidate-5-0-h74snzg5tpbh\" to finish creating: GET https://rp.j1843200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de4739f4-eebd-4fe2-bf80-3bf6d57ff617\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de4739f4-eebd-4fe2-bf80-3bf6d57ff617\",\n \"name\": \"de4739f4-eebd-4fe2-bf80-3bf6d57ff617\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T00:23:47.993778746Z\",\n \"endTime\": \"2026-08-25T00:42:49.076566651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-h74snzg5tpbh/cluster-candidate-5-0-gp9ztx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014a05c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-gp9ztx: failed waiting for cluster=\"cluster-candidate-5-0-gp9ztx\" in resourcegroup=\"rg-candidate-5-0-h74snzg5tpbh\" to finish creating: GET https://rp.j1843200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de4739f4-eebd-4fe2-bf80-3bf6d57ff617\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de4739f4-eebd-4fe2-bf80-3bf6d57ff617\",\n \"name\": \"de4739f4-eebd-4fe2-bf80-3bf6d57ff617\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-25T00:23:47.993778746Z\",\n \"endTime\": \"2026-08-25T00:42:49.076566651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092031971118026752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6648,"pr_state":"open","pr_sha":"d5a9f482aee0f501c8ae8bda8cd06ffa03214aaf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T23:30:58Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"e303f961fbb1779282cfa06c4b25bf401f4e819b34a545e11096cbc3df1f5529","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092031971118026752","occurred_at":"2026-08-24T23:30:58Z","failed_at":"e2e","signature_id":"176c7f8e0e9a6fe4e5a12f9055659eede136ab94bf2bc621167d758c6dc1828c","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00083c410\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[00:23:58.357]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:55.0351655Z);\\nlet _endTime = datetime(2026-08-25T00:23:55.0351661Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:23:58.3330999Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b\\\",\\r\\n \\\"activityId\\\": \\\"442abb20-5490-45f5-ba70-3bca3bed0639\\\",\\r\\n \\\"subActivityId\\\": \\\"b9132e8a-1c55-4f09-8982-f344852f0eee\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"99a04993-dace-423f-a879-2fbeee2a7b84\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b ARID=442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.Http.CallContext/442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.ExecuteQuery/99a04993-dace-423f-a879-2fbeee2a7b84 \\u003e DN.FE.ExecuteQuery/b9132e8a-1c55-4f09-8982-f344852f0eee)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:23:58.411]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:55.0351655Z);\\nlet _endTime = datetime(2026-08-25T00:23:55.0351661Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:23:58.3330999Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b\\\",\\r\\n \\\"activityId\\\": \\\"442abb20-5490-45f5-ba70-3bca3bed0639\\\",\\r\\n \\\"subActivityId\\\": \\\"b9132e8a-1c55-4f09-8982-f344852f0eee\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"99a04993-dace-423f-a879-2fbeee2a7b84\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b ARID=442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.Http.CallContext/442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.ExecuteQuery/99a04993-dace-423f-a879-2fbeee2a7b84 \\u003e DN.FE.ExecuteQuery/b9132e8a-1c55-4f09-8982-f344852f0eee)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[00:24:01.787]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:58.4821435Z);\\nlet _endTime = datetime(2026-08-25T00:23:58.4821442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:24:01.7618170Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5\\\",\\r\\n \\\"activityId\\\": \\\"d2adae1e-7609-4016-b166-5a8ff8be55d8\\\",\\r\\n \\\"subActivityId\\\": \\\"7c0e685c-e2d3-436f-89bc-9d9fa3141b61\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8cf560a3-f257-4d9d-900b-aaf46185fe8b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5 ARID=d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.Http.CallContext/d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.ExecuteQuery/8cf560a3-f257-4d9d-900b-aaf46185fe8b \\u003e DN.FE.ExecuteQuery/7c0e685c-e2d3-436f-89bc-9d9fa3141b61)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:24:01.845]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:58.4821435Z);\\nlet _endTime = datetime(2026-08-25T00:23:58.4821442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:24:01.7618170Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5\\\",\\r\\n \\\"activityId\\\": \\\"d2adae1e-7609-4016-b166-5a8ff8be55d8\\\",\\r\\n \\\"subActivityId\\\": \\\"7c0e685c-e2d3-436f-89bc-9d9fa3141b61\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8cf560a3-f257-4d9d-900b-aaf46185fe8b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5 ARID=d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.Http.CallContext/d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.ExecuteQuery/8cf560a3-f257-4d9d-900b-aaf46185fe8b \\u003e DN.FE.ExecuteQuery/7c0e685c-e2d3-436f-89bc-9d9fa3141b61)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[00:24:08.682]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:24:05.4122339Z);\\nlet _endTime = datetime(2026-08-25T00:24:05.4122344Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:24:08.6621602Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f\\\",\\r\\n \\\"activityId\\\": \\\"ed3e04f4-6a49-4e81-9b00-d4558d9cddcc\\\",\\r\\n \\\"subActivityId\\\": \\\"4cad9bba-6029-42b0-a77f-5c14bc4e8a84\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"58dfa59a-bdfb-430f-af23-104aaad94cc9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f ARID=ed3e04f4-6a49-4e81-9b00-d4558d9cddcc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/58dfa59a-bdfb-430f-af23-104aaad94cc9 \\u003e DN.FE.ExecuteQuery/4cad9bba-6029-42b0-a77f-5c14bc4e8a84)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:24:08.778]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:24:05.4122339Z);\\nlet _endTime = datetime(2026-08-25T00:24:05.4122344Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:24:08.6621602Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f\\\",\\r\\n \\\"activityId\\\": \\\"ed3e04f4-6a49-4e81-9b00-d4558d9cddcc\\\",\\r\\n \\\"subActivityId\\\": \\\"4cad9bba-6029-42b0-a77f-5c14bc4e8a84\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"58dfa59a-bdfb-430f-af23-104aaad94cc9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f ARID=ed3e04f4-6a49-4e81-9b00-d4558d9cddcc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/58dfa59a-bdfb-430f-af23-104aaad94cc9 \\u003e DN.FE.ExecuteQuery/4cad9bba-6029-42b0-a77f-5c14bc4e8a84)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00083c410\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[00:23:58.357]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:55.0351655Z);\\nlet _endTime = datetime(2026-08-25T00:23:55.0351661Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:23:58.3330999Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b\\\",\\r\\n \\\"activityId\\\": \\\"442abb20-5490-45f5-ba70-3bca3bed0639\\\",\\r\\n \\\"subActivityId\\\": \\\"b9132e8a-1c55-4f09-8982-f344852f0eee\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"99a04993-dace-423f-a879-2fbeee2a7b84\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b ARID=442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.Http.CallContext/442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.ExecuteQuery/99a04993-dace-423f-a879-2fbeee2a7b84 \\u003e DN.FE.ExecuteQuery/b9132e8a-1c55-4f09-8982-f344852f0eee)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:23:58.411]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:55.0351655Z);\\nlet _endTime = datetime(2026-08-25T00:23:55.0351661Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:23:58.3330999Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b\\\",\\r\\n \\\"activityId\\\": \\\"442abb20-5490-45f5-ba70-3bca3bed0639\\\",\\r\\n \\\"subActivityId\\\": \\\"b9132e8a-1c55-4f09-8982-f344852f0eee\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"99a04993-dace-423f-a879-2fbeee2a7b84\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b ARID=442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.Http.CallContext/442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.ExecuteQuery/99a04993-dace-423f-a879-2fbeee2a7b84 \\u003e DN.FE.ExecuteQuery/b9132e8a-1c55-4f09-8982-f344852f0eee)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[00:24:01.787]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:58.4821435Z);\\nlet _endTime = datetime(2026-08-25T00:23:58.4821442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:24:01.7618170Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5\\\",\\r\\n \\\"activityId\\\": \\\"d2adae1e-7609-4016-b166-5a8ff8be55d8\\\",\\r\\n \\\"subActivityId\\\": \\\"7c0e685c-e2d3-436f-89bc-9d9fa3141b61\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8cf560a3-f257-4d9d-900b-aaf46185fe8b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5 ARID=d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.Http.CallContext/d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.ExecuteQuery/8cf560a3-f257-4d9d-900b-aaf46185fe8b \\u003e DN.FE.ExecuteQuery/7c0e685c-e2d3-436f-89bc-9d9fa3141b61)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:24:01.845]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:58.4821435Z);\\nlet _endTime = datetime(2026-08-25T00:23:58.4821442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:24:01.7618170Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5\\\",\\r\\n \\\"activityId\\\": \\\"d2adae1e-7609-4016-b166-5a8ff8be55d8\\\",\\r\\n \\\"subActivityId\\\": \\\"7c0e685c-e2d3-436f-89bc-9d9fa3141b61\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8cf560a3-f257-4d9d-900b-aaf46185fe8b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5 ARID=d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.Http.CallContext/d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.ExecuteQuery/8cf560a3-f257-4d9d-900b-aaf46185fe8b \\u003e DN.FE.ExecuteQuery/7c0e685c-e2d3-436f-89bc-9d9fa3141b61)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[00:24:08.682]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:24:05.4122339Z);\\nlet _endTime = datetime(2026-08-25T00:24:05.4122344Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:24:08.6621602Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f\\\",\\r\\n \\\"activityId\\\": \\\"ed3e04f4-6a49-4e81-9b00-d4558d9cddcc\\\",\\r\\n \\\"subActivityId\\\": \\\"4cad9bba-6029-42b0-a77f-5c14bc4e8a84\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"58dfa59a-bdfb-430f-af23-104aaad94cc9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f ARID=ed3e04f4-6a49-4e81-9b00-d4558d9cddcc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/58dfa59a-bdfb-430f-af23-104aaad94cc9 \\u003e DN.FE.ExecuteQuery/4cad9bba-6029-42b0-a77f-5c14bc4e8a84)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:24:08.778]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:24:05.4122339Z);\\nlet _endTime = datetime(2026-08-25T00:24:05.4122344Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:24:08.6621602Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f\\\",\\r\\n \\\"activityId\\\": \\\"ed3e04f4-6a49-4e81-9b00-d4558d9cddcc\\\",\\r\\n \\\"subActivityId\\\": \\\"4cad9bba-6029-42b0-a77f-5c14bc4e8a84\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"58dfa59a-bdfb-430f-af23-104aaad94cc9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f ARID=ed3e04f4-6a49-4e81-9b00-d4558d9cddcc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/58dfa59a-bdfb-430f-af23-104aaad94cc9 \\u003e DN.FE.ExecuteQuery/4cad9bba-6029-42b0-a77f-5c14bc4e8a84)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"fd07094b91ba0e0975ac3095332f4464f82e8aa1a6fb869f3a588fb821ca9d57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092031971118026752","occurred_at":"2026-08-24T23:30:58Z","failed_at":"alert","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-25T00:10:04Z\n Ended: 2026-08-25T00:34:04Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j8026752-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092030482601480192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6660,"pr_state":"open","pr_sha":"a660b6e55fc0b5bbac8bc93588a1797519a9c76f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T23:25:09Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7e74b0ae172757048aa81d9d15a6769f3ca5845eae9258bb245d036b3c44339d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092030482601480192","occurred_at":"2026-08-24T23:25:09Z","failed_at":"e2e","signature_id":"a4d769910ea57a048ae6030e8851eaac93f508a9d62968944253d616d8fe88ed","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000626a40\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[00:14:56.798]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:53.4068554Z);\\nlet _endTime = datetime(2026-08-25T00:14:53.4068559Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:14:56.7779164Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d\\\",\\r\\n \\\"activityId\\\": \\\"697030e7-6169-48d4-afab-e55af5423283\\\",\\r\\n \\\"subActivityId\\\": \\\"7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e5629797-dc74-431c-89f2-26cc2ad935e1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d ARID=697030e7-6169-48d4-afab-e55af5423283 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5629797-dc74-431c-89f2-26cc2ad935e1 \\u003e DN.FE.ExecuteQuery/7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:14:56.846]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:53.4068554Z);\\nlet _endTime = datetime(2026-08-25T00:14:53.4068559Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:14:56.7779164Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d\\\",\\r\\n \\\"activityId\\\": \\\"697030e7-6169-48d4-afab-e55af5423283\\\",\\r\\n \\\"subActivityId\\\": \\\"7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e5629797-dc74-431c-89f2-26cc2ad935e1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d ARID=697030e7-6169-48d4-afab-e55af5423283 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5629797-dc74-431c-89f2-26cc2ad935e1 \\u003e DN.FE.ExecuteQuery/7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[00:15:00.265]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:56.9537642Z);\\nlet _endTime = datetime(2026-08-25T00:14:56.9537648Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:00.2458657Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130\\\",\\r\\n \\\"activityId\\\": \\\"60865211-a5af-4242-8bd8-14b7a1f199ba\\\",\\r\\n \\\"subActivityId\\\": \\\"865c926e-620f-4107-9fd8-43ac9ee494b3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"70a56a06-7a14-4b6c-b645-990ed51e93ae\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130 ARID=60865211-a5af-4242-8bd8-14b7a1f199ba \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/70a56a06-7a14-4b6c-b645-990ed51e93ae \\u003e DN.FE.ExecuteQuery/865c926e-620f-4107-9fd8-43ac9ee494b3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:15:00.327]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:56.9537642Z);\\nlet _endTime = datetime(2026-08-25T00:14:56.9537648Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:00.2458657Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130\\\",\\r\\n \\\"activityId\\\": \\\"60865211-a5af-4242-8bd8-14b7a1f199ba\\\",\\r\\n \\\"subActivityId\\\": \\\"865c926e-620f-4107-9fd8-43ac9ee494b3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"70a56a06-7a14-4b6c-b645-990ed51e93ae\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130 ARID=60865211-a5af-4242-8bd8-14b7a1f199ba \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/70a56a06-7a14-4b6c-b645-990ed51e93ae \\u003e DN.FE.ExecuteQuery/865c926e-620f-4107-9fd8-43ac9ee494b3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[00:15:08.703]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:04.4342198Z);\\nlet _endTime = datetime(2026-08-25T00:15:04.4342235Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:08.6768100Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199\\\",\\r\\n \\\"activityId\\\": \\\"a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19\\\",\\r\\n \\\"subActivityId\\\": \\\"dda68eb2-c740-498f-b62e-c74a25b621f7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4d4ca23c-dcca-4b88-9136-446102508ccb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199 ARID=a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.Http.CallContext/a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.ExecuteQuery/4d4ca23c-dcca-4b88-9136-446102508ccb \\u003e DN.FE.ExecuteQuery/dda68eb2-c740-498f-b62e-c74a25b621f7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:15:08.705]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"context canceled\",\n \"name\": \"cosmosResourceSnapshots\"\n }�[0m\n �[37m[00:15:09.096]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:04.4342198Z);\\nlet _endTime = datetime(2026-08-25T00:15:04.4342235Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:08.6768100Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199\\\",\\r\\n \\\"activityId\\\": \\\"a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19\\\",\\r\\n \\\"subActivityId\\\": \\\"dda68eb2-c740-498f-b62e-c74a25b621f7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4d4ca23c-dcca-4b88-9136-446102508ccb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199 ARID=a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.Http.CallContext/a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.ExecuteQuery/4d4ca23c-dcca-4b88-9136-446102508ccb \\u003e DN.FE.ExecuteQuery/dda68eb2-c740-498f-b62e-c74a25b621f7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000626a40\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[00:14:56.798]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:53.4068554Z);\\nlet _endTime = datetime(2026-08-25T00:14:53.4068559Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:14:56.7779164Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d\\\",\\r\\n \\\"activityId\\\": \\\"697030e7-6169-48d4-afab-e55af5423283\\\",\\r\\n \\\"subActivityId\\\": \\\"7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e5629797-dc74-431c-89f2-26cc2ad935e1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d ARID=697030e7-6169-48d4-afab-e55af5423283 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5629797-dc74-431c-89f2-26cc2ad935e1 \\u003e DN.FE.ExecuteQuery/7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:14:56.846]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:53.4068554Z);\\nlet _endTime = datetime(2026-08-25T00:14:53.4068559Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:14:56.7779164Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d\\\",\\r\\n \\\"activityId\\\": \\\"697030e7-6169-48d4-afab-e55af5423283\\\",\\r\\n \\\"subActivityId\\\": \\\"7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e5629797-dc74-431c-89f2-26cc2ad935e1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d ARID=697030e7-6169-48d4-afab-e55af5423283 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5629797-dc74-431c-89f2-26cc2ad935e1 \\u003e DN.FE.ExecuteQuery/7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[00:15:00.265]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:56.9537642Z);\\nlet _endTime = datetime(2026-08-25T00:14:56.9537648Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:00.2458657Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130\\\",\\r\\n \\\"activityId\\\": \\\"60865211-a5af-4242-8bd8-14b7a1f199ba\\\",\\r\\n \\\"subActivityId\\\": \\\"865c926e-620f-4107-9fd8-43ac9ee494b3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"70a56a06-7a14-4b6c-b645-990ed51e93ae\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130 ARID=60865211-a5af-4242-8bd8-14b7a1f199ba \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/70a56a06-7a14-4b6c-b645-990ed51e93ae \\u003e DN.FE.ExecuteQuery/865c926e-620f-4107-9fd8-43ac9ee494b3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:15:00.327]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:56.9537642Z);\\nlet _endTime = datetime(2026-08-25T00:14:56.9537648Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:00.2458657Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130\\\",\\r\\n \\\"activityId\\\": \\\"60865211-a5af-4242-8bd8-14b7a1f199ba\\\",\\r\\n \\\"subActivityId\\\": \\\"865c926e-620f-4107-9fd8-43ac9ee494b3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"70a56a06-7a14-4b6c-b645-990ed51e93ae\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130 ARID=60865211-a5af-4242-8bd8-14b7a1f199ba \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/70a56a06-7a14-4b6c-b645-990ed51e93ae \\u003e DN.FE.ExecuteQuery/865c926e-620f-4107-9fd8-43ac9ee494b3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[00:15:08.703]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:04.4342198Z);\\nlet _endTime = datetime(2026-08-25T00:15:04.4342235Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:08.6768100Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199\\\",\\r\\n \\\"activityId\\\": \\\"a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19\\\",\\r\\n \\\"subActivityId\\\": \\\"dda68eb2-c740-498f-b62e-c74a25b621f7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4d4ca23c-dcca-4b88-9136-446102508ccb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199 ARID=a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.Http.CallContext/a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.ExecuteQuery/4d4ca23c-dcca-4b88-9136-446102508ccb \\u003e DN.FE.ExecuteQuery/dda68eb2-c740-498f-b62e-c74a25b621f7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:15:08.705]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"context canceled\",\n \"name\": \"cosmosResourceSnapshots\"\n }�[0m\n �[37m[00:15:09.096]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:04.4342198Z);\\nlet _endTime = datetime(2026-08-25T00:15:04.4342235Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:08.6768100Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199\\\",\\r\\n \\\"activityId\\\": \\\"a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19\\\",\\r\\n \\\"subActivityId\\\": \\\"dda68eb2-c740-498f-b62e-c74a25b621f7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4d4ca23c-dcca-4b88-9136-446102508ccb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199 ARID=a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.Http.CallContext/a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.ExecuteQuery/4d4ca23c-dcca-4b88-9136-446102508ccb \\u003e DN.FE.ExecuteQuery/dda68eb2-c740-498f-b62e-c74a25b621f7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092025885698822144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6504,"pr_state":"open","pr_sha":"af0ab1f037eef8141b97aadd390cc54296b6d1c3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T23:06:53Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bb653b57da26d3c3dc7815c2e9e2ab10f4c9888dc7dd6b921202f0fbb30ce027","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092025885698822144","occurred_at":"2026-08-24T23:06:53Z","failed_at":"e2e","signature_id":"9eb8d09c21becab11e4df2e2486fbadfcfe682a12484ff5957cc08b54e0cd902","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.006s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001f4a670\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[00:15:10.975]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:05.9070266Z);\\nlet _endTime = datetime(2026-08-25T00:15:05.9070271Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hl4wgvvqgzxr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:10.9519459Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;26675931-f7e7-4344-9b85-24a650b39392\\\",\\r\\n \\\"activityId\\\": \\\"ef76707e-b3d2-4ccf-b2b7-9437b788ee47\\\",\\r\\n \\\"subActivityId\\\": \\\"d1814cb1-6831-44dc-8356-c144fb092d85\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ee35d653-7145-4b4c-9e8d-2c7d4c3f2774\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;26675931-f7e7-4344-9b85-24a650b39392 ARID=ef76707e-b3d2-4ccf-b2b7-9437b788ee47 \\u003e GW.Http.CallContext/ef76707e-b3d2-4ccf-b2b7-9437b788ee47 \\u003e GW.ExecuteQuery/ee35d653-7145-4b4c-9e8d-2c7d4c3f2774 \\u003e DN.FE.ExecuteQuery/d1814cb1-6831-44dc-8356-c144fb092d85)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:15:11.171]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:05.9070266Z);\\nlet _endTime = datetime(2026-08-25T00:15:05.9070271Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hl4wgvvqgzxr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:10.9519459Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;26675931-f7e7-4344-9b85-24a650b39392\\\",\\r\\n \\\"activityId\\\": \\\"ef76707e-b3d2-4ccf-b2b7-9437b788ee47\\\",\\r\\n \\\"subActivityId\\\": \\\"d1814cb1-6831-44dc-8356-c144fb092d85\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ee35d653-7145-4b4c-9e8d-2c7d4c3f2774\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;26675931-f7e7-4344-9b85-24a650b39392 ARID=ef76707e-b3d2-4ccf-b2b7-9437b788ee47 \\u003e GW.Http.CallContext/ef76707e-b3d2-4ccf-b2b7-9437b788ee47 \\u003e GW.ExecuteQuery/ee35d653-7145-4b4c-9e8d-2c7d4c3f2774 \\u003e DN.FE.ExecuteQuery/d1814cb1-6831-44dc-8356-c144fb092d85)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[00:15:15.030]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:11.2948478Z);\\nlet _endTime = datetime(2026-08-25T00:15:11.2948504Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hl4wgvvqgzxr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:15.0090681Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b3257560-3cb6-4fcc-8e7f-ed9367adc845\\\",\\r\\n \\\"activityId\\\": \\\"c0adf2ff-92d7-4876-9381-3d81d9c09dcf\\\",\\r\\n \\\"subActivityId\\\": \\\"d36c0b97-879a-48b5-9161-c4d48dd18d09\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"a014e706-b98c-4023-88e5-6de86b03b1af\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b3257560-3cb6-4fcc-8e7f-ed9367adc845 ARID=c0adf2ff-92d7-4876-9381-3d81d9c09dcf \\u003e GW.Http.CallContext/c0adf2ff-92d7-4876-9381-3d81d9c09dcf \\u003e GW.ExecuteQuery/a014e706-b98c-4023-88e5-6de86b03b1af \\u003e DN.FE.ExecuteQuery/d36c0b97-879a-48b5-9161-c4d48dd18d09)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:15:15.124]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:11.2948478Z);\\nlet _endTime = datetime(2026-08-25T00:15:11.2948504Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hl4wgvvqgzxr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:15.0090681Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b3257560-3cb6-4fcc-8e7f-ed9367adc845\\\",\\r\\n \\\"activityId\\\": \\\"c0adf2ff-92d7-4876-9381-3d81d9c09dcf\\\",\\r\\n \\\"subActivityId\\\": \\\"d36c0b97-879a-48b5-9161-c4d48dd18d09\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"a014e706-b98c-4023-88e5-6de86b03b1af\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b3257560-3cb6-4fcc-8e7f-ed9367adc845 ARID=c0adf2ff-92d7-4876-9381-3d81d9c09dcf \\u003e GW.Http.CallContext/c0adf2ff-92d7-4876-9381-3d81d9c09dcf \\u003e GW.ExecuteQuery/a014e706-b98c-4023-88e5-6de86b03b1af \\u003e DN.FE.ExecuteQuery/d36c0b97-879a-48b5-9161-c4d48dd18d09)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[00:15:22.332]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:19.0004961Z);\\nlet _endTime = datetime(2026-08-25T00:15:19.0004966Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hl4wgvvqgzxr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:22.3098397Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b394207c-245e-45d5-a804-f89ebe5bae42\\\",\\r\\n \\\"activityId\\\": \\\"00780658-87c4-4f5e-a63b-4d7f35817d45\\\",\\r\\n \\\"subActivityId\\\": \\\"08e72ece-2adb-4757-8239-b00748f8c803\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4e923dde-12b5-4490-b57b-833a13d432e8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b394207c-245e-45d5-a804-f89ebe5bae42 ARID=00780658-87c4-4f5e-a63b-4d7f35817d45 \\u003e GW.Http.CallContext/00780658-87c4-4f5e-a63b-4d7f35817d45 \\u003e GW.ExecuteQuery/4e923dde-12b5-4490-b57b-833a13d432e8 \\u003e DN.FE.ExecuteQuery/08e72ece-2adb-4757-8239-b00748f8c803)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:15:22.436]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:19.0004961Z);\\nlet _endTime = datetime(2026-08-25T00:15:19.0004966Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hl4wgvvqgzxr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:22.3098397Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b394207c-245e-45d5-a804-f89ebe5bae42\\\",\\r\\n \\\"activityId\\\": \\\"00780658-87c4-4f5e-a63b-4d7f35817d45\\\",\\r\\n \\\"subActivityId\\\": \\\"08e72ece-2adb-4757-8239-b00748f8c803\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4e923dde-12b5-4490-b57b-833a13d432e8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b394207c-245e-45d5-a804-f89ebe5bae42 ARID=00780658-87c4-4f5e-a63b-4d7f35817d45 \\u003e GW.Http.CallContext/00780658-87c4-4f5e-a63b-4d7f35817d45 \\u003e GW.ExecuteQuery/4e923dde-12b5-4490-b57b-833a13d432e8 \\u003e DN.FE.ExecuteQuery/08e72ece-2adb-4757-8239-b00748f8c803)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.006s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001f4a670\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[00:15:10.975]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:05.9070266Z);\\nlet _endTime = datetime(2026-08-25T00:15:05.9070271Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hl4wgvvqgzxr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:10.9519459Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;26675931-f7e7-4344-9b85-24a650b39392\\\",\\r\\n \\\"activityId\\\": \\\"ef76707e-b3d2-4ccf-b2b7-9437b788ee47\\\",\\r\\n \\\"subActivityId\\\": \\\"d1814cb1-6831-44dc-8356-c144fb092d85\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ee35d653-7145-4b4c-9e8d-2c7d4c3f2774\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;26675931-f7e7-4344-9b85-24a650b39392 ARID=ef76707e-b3d2-4ccf-b2b7-9437b788ee47 \\u003e GW.Http.CallContext/ef76707e-b3d2-4ccf-b2b7-9437b788ee47 \\u003e GW.ExecuteQuery/ee35d653-7145-4b4c-9e8d-2c7d4c3f2774 \\u003e DN.FE.ExecuteQuery/d1814cb1-6831-44dc-8356-c144fb092d85)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:15:11.171]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:05.9070266Z);\\nlet _endTime = datetime(2026-08-25T00:15:05.9070271Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hl4wgvvqgzxr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:10.9519459Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;26675931-f7e7-4344-9b85-24a650b39392\\\",\\r\\n \\\"activityId\\\": \\\"ef76707e-b3d2-4ccf-b2b7-9437b788ee47\\\",\\r\\n \\\"subActivityId\\\": \\\"d1814cb1-6831-44dc-8356-c144fb092d85\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ee35d653-7145-4b4c-9e8d-2c7d4c3f2774\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;26675931-f7e7-4344-9b85-24a650b39392 ARID=ef76707e-b3d2-4ccf-b2b7-9437b788ee47 \\u003e GW.Http.CallContext/ef76707e-b3d2-4ccf-b2b7-9437b788ee47 \\u003e GW.ExecuteQuery/ee35d653-7145-4b4c-9e8d-2c7d4c3f2774 \\u003e DN.FE.ExecuteQuery/d1814cb1-6831-44dc-8356-c144fb092d85)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[00:15:15.030]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:11.2948478Z);\\nlet _endTime = datetime(2026-08-25T00:15:11.2948504Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hl4wgvvqgzxr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:15.0090681Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b3257560-3cb6-4fcc-8e7f-ed9367adc845\\\",\\r\\n \\\"activityId\\\": \\\"c0adf2ff-92d7-4876-9381-3d81d9c09dcf\\\",\\r\\n \\\"subActivityId\\\": \\\"d36c0b97-879a-48b5-9161-c4d48dd18d09\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"a014e706-b98c-4023-88e5-6de86b03b1af\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b3257560-3cb6-4fcc-8e7f-ed9367adc845 ARID=c0adf2ff-92d7-4876-9381-3d81d9c09dcf \\u003e GW.Http.CallContext/c0adf2ff-92d7-4876-9381-3d81d9c09dcf \\u003e GW.ExecuteQuery/a014e706-b98c-4023-88e5-6de86b03b1af \\u003e DN.FE.ExecuteQuery/d36c0b97-879a-48b5-9161-c4d48dd18d09)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:15:15.124]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:11.2948478Z);\\nlet _endTime = datetime(2026-08-25T00:15:11.2948504Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hl4wgvvqgzxr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:15.0090681Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b3257560-3cb6-4fcc-8e7f-ed9367adc845\\\",\\r\\n \\\"activityId\\\": \\\"c0adf2ff-92d7-4876-9381-3d81d9c09dcf\\\",\\r\\n \\\"subActivityId\\\": \\\"d36c0b97-879a-48b5-9161-c4d48dd18d09\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"a014e706-b98c-4023-88e5-6de86b03b1af\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b3257560-3cb6-4fcc-8e7f-ed9367adc845 ARID=c0adf2ff-92d7-4876-9381-3d81d9c09dcf \\u003e GW.Http.CallContext/c0adf2ff-92d7-4876-9381-3d81d9c09dcf \\u003e GW.ExecuteQuery/a014e706-b98c-4023-88e5-6de86b03b1af \\u003e DN.FE.ExecuteQuery/d36c0b97-879a-48b5-9161-c4d48dd18d09)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[00:15:22.332]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:19.0004961Z);\\nlet _endTime = datetime(2026-08-25T00:15:19.0004966Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hl4wgvvqgzxr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:22.3098397Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b394207c-245e-45d5-a804-f89ebe5bae42\\\",\\r\\n \\\"activityId\\\": \\\"00780658-87c4-4f5e-a63b-4d7f35817d45\\\",\\r\\n \\\"subActivityId\\\": \\\"08e72ece-2adb-4757-8239-b00748f8c803\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4e923dde-12b5-4490-b57b-833a13d432e8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b394207c-245e-45d5-a804-f89ebe5bae42 ARID=00780658-87c4-4f5e-a63b-4d7f35817d45 \\u003e GW.Http.CallContext/00780658-87c4-4f5e-a63b-4d7f35817d45 \\u003e GW.ExecuteQuery/4e923dde-12b5-4490-b57b-833a13d432e8 \\u003e DN.FE.ExecuteQuery/08e72ece-2adb-4757-8239-b00748f8c803)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:15:22.436]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:19.0004961Z);\\nlet _endTime = datetime(2026-08-25T00:15:19.0004966Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hl4wgvvqgzxr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:15:22.3098397Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b394207c-245e-45d5-a804-f89ebe5bae42\\\",\\r\\n \\\"activityId\\\": \\\"00780658-87c4-4f5e-a63b-4d7f35817d45\\\",\\r\\n \\\"subActivityId\\\": \\\"08e72ece-2adb-4757-8239-b00748f8c803\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4e923dde-12b5-4490-b57b-833a13d432e8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b394207c-245e-45d5-a804-f89ebe5bae42 ARID=00780658-87c4-4f5e-a63b-4d7f35817d45 \\u003e GW.Http.CallContext/00780658-87c4-4f5e-a63b-4d7f35817d45 \\u003e GW.ExecuteQuery/4e923dde-12b5-4490-b57b-833a13d432e8 \\u003e DN.FE.ExecuteQuery/08e72ece-2adb-4757-8239-b00748f8c803)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6635,"pr_state":"open","pr_sha":"aa2fb7c461dcc2c4085b2ff07f5627ab9885ba63","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T22:44:28Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"09475ba05790bbc9cd683b30d8aa4449e38940f665167455dae12aedf2db2ef8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","failed_at":"e2e","signature_id":"e6a9db1c5a3cafd8f545d8476a1e6b49dd9166a98aa20d59709936d40b1c52c0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-x45vwlgpgvzk/cluster-candidate-4-21-rztpw4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3e6a0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-rztpw4: failed waiting for cluster=\"cluster-candidate-4-21-rztpw4\" in resourcegroup=\"rg-candidate-4-21-x45vwlgpgvzk\" to finish creating: GET https://rp.j4709760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e986c420-0503-4186-9a01-bad1f92a005c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e986c420-0503-4186-9a01-bad1f92a005c\",\n \"name\": \"e986c420-0503-4186-9a01-bad1f92a005c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:37:07.202301694Z\",\n \"endTime\": \"2026-08-24T23:56:10.677598487Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-x45vwlgpgvzk/cluster-candidate-4-21-rztpw4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3e6a0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-rztpw4: failed waiting for cluster=\"cluster-candidate-4-21-rztpw4\" in resourcegroup=\"rg-candidate-4-21-x45vwlgpgvzk\" to finish creating: GET https://rp.j4709760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e986c420-0503-4186-9a01-bad1f92a005c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e986c420-0503-4186-9a01-bad1f92a005c\",\n \"name\": \"e986c420-0503-4186-9a01-bad1f92a005c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:37:07.202301694Z\",\n \"endTime\": \"2026-08-24T23:56:10.677598487Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ab8d30e3ad7230eb3767b811cfff63971de39fe230199ccc997873a83c3762b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","failed_at":"e2e","signature_id":"a99ad7aa848b9f85d32a71fb35dfd49c3d8270fdc830d4478b6af42e4702a425","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0007c8410\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[23:59:56.120]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:59:52.8342332Z);\\nlet _endTime = datetime(2026-08-24T23:59:52.8342339Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6pmwg8htdhg9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:59:56.0986998Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;589862cc-bbb1-4912-808c-32c7239931d3\\\",\\r\\n \\\"activityId\\\": \\\"25c493b7-b925-4f70-a720-29da9868f2ec\\\",\\r\\n \\\"subActivityId\\\": \\\"77734989-d965-4eea-ad4e-90e589bfc994\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"03f18cdd-a350-4699-9b6f-326a47f2842c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;589862cc-bbb1-4912-808c-32c7239931d3 ARID=25c493b7-b925-4f70-a720-29da9868f2ec \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4bbda136-b57d-4b2b-810c-8cb7b1bbdba1 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4bbda136-b57d-4b2b-810c-8cb7b1bbdba1 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/03f18cdd-a350-4699-9b6f-326a47f2842c \\u003e DN.FE.ExecuteQuery/77734989-d965-4eea-ad4e-90e589bfc994)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:59:56.169]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:59:52.8342332Z);\\nlet _endTime = datetime(2026-08-24T23:59:52.8342339Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6pmwg8htdhg9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:59:56.0986998Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;589862cc-bbb1-4912-808c-32c7239931d3\\\",\\r\\n \\\"activityId\\\": \\\"25c493b7-b925-4f70-a720-29da9868f2ec\\\",\\r\\n \\\"subActivityId\\\": \\\"77734989-d965-4eea-ad4e-90e589bfc994\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"03f18cdd-a350-4699-9b6f-326a47f2842c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;589862cc-bbb1-4912-808c-32c7239931d3 ARID=25c493b7-b925-4f70-a720-29da9868f2ec \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4bbda136-b57d-4b2b-810c-8cb7b1bbdba1 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4bbda136-b57d-4b2b-810c-8cb7b1bbdba1 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/03f18cdd-a350-4699-9b6f-326a47f2842c \\u003e DN.FE.ExecuteQuery/77734989-d965-4eea-ad4e-90e589bfc994)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[23:59:59.562]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:59:56.2555369Z);\\nlet _endTime = datetime(2026-08-24T23:59:56.2555377Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6pmwg8htdhg9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:59:59.5412585Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8bde4e19-6492-4fdd-8666-87105fc1bc29\\\",\\r\\n \\\"activityId\\\": \\\"2dd85a88-1f1b-4ca3-a627-624434e67a6e\\\",\\r\\n \\\"subActivityId\\\": \\\"7321b56c-2660-40dd-827d-5978e678ae28\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b051aa4d-89e0-4106-a0dd-7ed70662f0f5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8bde4e19-6492-4fdd-8666-87105fc1bc29 ARID=2dd85a88-1f1b-4ca3-a627-624434e67a6e \\u003e GW.Http.CallContext/2dd85a88-1f1b-4ca3-a627-624434e67a6e \\u003e GW.ExecuteQuery/b051aa4d-89e0-4106-a0dd-7ed70662f0f5 \\u003e DN.FE.ExecuteQuery/7321b56c-2660-40dd-827d-5978e678ae28)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:59:59.611]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:59:56.2555369Z);\\nlet _endTime = datetime(2026-08-24T23:59:56.2555377Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6pmwg8htdhg9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:59:59.5412585Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8bde4e19-6492-4fdd-8666-87105fc1bc29\\\",\\r\\n \\\"activityId\\\": \\\"2dd85a88-1f1b-4ca3-a627-624434e67a6e\\\",\\r\\n \\\"subActivityId\\\": \\\"7321b56c-2660-40dd-827d-5978e678ae28\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b051aa4d-89e0-4106-a0dd-7ed70662f0f5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8bde4e19-6492-4fdd-8666-87105fc1bc29 ARID=2dd85a88-1f1b-4ca3-a627-624434e67a6e \\u003e GW.Http.CallContext/2dd85a88-1f1b-4ca3-a627-624434e67a6e \\u003e GW.ExecuteQuery/b051aa4d-89e0-4106-a0dd-7ed70662f0f5 \\u003e DN.FE.ExecuteQuery/7321b56c-2660-40dd-827d-5978e678ae28)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[00:00:06.688]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:00:03.1874189Z);\\nlet _endTime = datetime(2026-08-25T00:00:03.1874200Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6pmwg8htdhg9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:00:06.6685554Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8161fae6-5ba5-437e-9969-ded5f2fa4099\\\",\\r\\n \\\"activityId\\\": \\\"f6ea14a6-45ea-406a-84a9-e4660aa2592c\\\",\\r\\n \\\"subActivityId\\\": \\\"a15ff010-b600-4c98-b9b4-eec67c625699\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"95b62cbe-f648-4de9-8ee1-66a05ab19238\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8161fae6-5ba5-437e-9969-ded5f2fa4099 ARID=f6ea14a6-45ea-406a-84a9-e4660aa2592c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4a46171d-4922-4ebe-a2fb-926ed44edad6 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4a46171d-4922-4ebe-a2fb-926ed44edad6 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/95b62cbe-f648-4de9-8ee1-66a05ab19238 \\u003e DN.FE.ExecuteQuery/a15ff010-b600-4c98-b9b4-eec67c625699)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:00:06.794]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:00:03.1874189Z);\\nlet _endTime = datetime(2026-08-25T00:00:03.1874200Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6pmwg8htdhg9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:00:06.6685554Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8161fae6-5ba5-437e-9969-ded5f2fa4099\\\",\\r\\n \\\"activityId\\\": \\\"f6ea14a6-45ea-406a-84a9-e4660aa2592c\\\",\\r\\n \\\"subActivityId\\\": \\\"a15ff010-b600-4c98-b9b4-eec67c625699\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"95b62cbe-f648-4de9-8ee1-66a05ab19238\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8161fae6-5ba5-437e-9969-ded5f2fa4099 ARID=f6ea14a6-45ea-406a-84a9-e4660aa2592c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4a46171d-4922-4ebe-a2fb-926ed44edad6 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4a46171d-4922-4ebe-a2fb-926ed44edad6 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/95b62cbe-f648-4de9-8ee1-66a05ab19238 \\u003e DN.FE.ExecuteQuery/a15ff010-b600-4c98-b9b4-eec67c625699)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0007c8410\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[23:59:56.120]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:59:52.8342332Z);\\nlet _endTime = datetime(2026-08-24T23:59:52.8342339Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6pmwg8htdhg9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:59:56.0986998Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;589862cc-bbb1-4912-808c-32c7239931d3\\\",\\r\\n \\\"activityId\\\": \\\"25c493b7-b925-4f70-a720-29da9868f2ec\\\",\\r\\n \\\"subActivityId\\\": \\\"77734989-d965-4eea-ad4e-90e589bfc994\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"03f18cdd-a350-4699-9b6f-326a47f2842c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;589862cc-bbb1-4912-808c-32c7239931d3 ARID=25c493b7-b925-4f70-a720-29da9868f2ec \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4bbda136-b57d-4b2b-810c-8cb7b1bbdba1 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4bbda136-b57d-4b2b-810c-8cb7b1bbdba1 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/03f18cdd-a350-4699-9b6f-326a47f2842c \\u003e DN.FE.ExecuteQuery/77734989-d965-4eea-ad4e-90e589bfc994)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:59:56.169]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:59:52.8342332Z);\\nlet _endTime = datetime(2026-08-24T23:59:52.8342339Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6pmwg8htdhg9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:59:56.0986998Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;589862cc-bbb1-4912-808c-32c7239931d3\\\",\\r\\n \\\"activityId\\\": \\\"25c493b7-b925-4f70-a720-29da9868f2ec\\\",\\r\\n \\\"subActivityId\\\": \\\"77734989-d965-4eea-ad4e-90e589bfc994\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"03f18cdd-a350-4699-9b6f-326a47f2842c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;589862cc-bbb1-4912-808c-32c7239931d3 ARID=25c493b7-b925-4f70-a720-29da9868f2ec \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4bbda136-b57d-4b2b-810c-8cb7b1bbdba1 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4bbda136-b57d-4b2b-810c-8cb7b1bbdba1 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/03f18cdd-a350-4699-9b6f-326a47f2842c \\u003e DN.FE.ExecuteQuery/77734989-d965-4eea-ad4e-90e589bfc994)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[23:59:59.562]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:59:56.2555369Z);\\nlet _endTime = datetime(2026-08-24T23:59:56.2555377Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6pmwg8htdhg9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:59:59.5412585Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8bde4e19-6492-4fdd-8666-87105fc1bc29\\\",\\r\\n \\\"activityId\\\": \\\"2dd85a88-1f1b-4ca3-a627-624434e67a6e\\\",\\r\\n \\\"subActivityId\\\": \\\"7321b56c-2660-40dd-827d-5978e678ae28\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b051aa4d-89e0-4106-a0dd-7ed70662f0f5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8bde4e19-6492-4fdd-8666-87105fc1bc29 ARID=2dd85a88-1f1b-4ca3-a627-624434e67a6e \\u003e GW.Http.CallContext/2dd85a88-1f1b-4ca3-a627-624434e67a6e \\u003e GW.ExecuteQuery/b051aa4d-89e0-4106-a0dd-7ed70662f0f5 \\u003e DN.FE.ExecuteQuery/7321b56c-2660-40dd-827d-5978e678ae28)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:59:59.611]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:59:56.2555369Z);\\nlet _endTime = datetime(2026-08-24T23:59:56.2555377Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6pmwg8htdhg9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:59:59.5412585Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8bde4e19-6492-4fdd-8666-87105fc1bc29\\\",\\r\\n \\\"activityId\\\": \\\"2dd85a88-1f1b-4ca3-a627-624434e67a6e\\\",\\r\\n \\\"subActivityId\\\": \\\"7321b56c-2660-40dd-827d-5978e678ae28\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b051aa4d-89e0-4106-a0dd-7ed70662f0f5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8bde4e19-6492-4fdd-8666-87105fc1bc29 ARID=2dd85a88-1f1b-4ca3-a627-624434e67a6e \\u003e GW.Http.CallContext/2dd85a88-1f1b-4ca3-a627-624434e67a6e \\u003e GW.ExecuteQuery/b051aa4d-89e0-4106-a0dd-7ed70662f0f5 \\u003e DN.FE.ExecuteQuery/7321b56c-2660-40dd-827d-5978e678ae28)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[00:00:06.688]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:00:03.1874189Z);\\nlet _endTime = datetime(2026-08-25T00:00:03.1874200Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6pmwg8htdhg9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:00:06.6685554Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8161fae6-5ba5-437e-9969-ded5f2fa4099\\\",\\r\\n \\\"activityId\\\": \\\"f6ea14a6-45ea-406a-84a9-e4660aa2592c\\\",\\r\\n \\\"subActivityId\\\": \\\"a15ff010-b600-4c98-b9b4-eec67c625699\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"95b62cbe-f648-4de9-8ee1-66a05ab19238\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8161fae6-5ba5-437e-9969-ded5f2fa4099 ARID=f6ea14a6-45ea-406a-84a9-e4660aa2592c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4a46171d-4922-4ebe-a2fb-926ed44edad6 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4a46171d-4922-4ebe-a2fb-926ed44edad6 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/95b62cbe-f648-4de9-8ee1-66a05ab19238 \\u003e DN.FE.ExecuteQuery/a15ff010-b600-4c98-b9b4-eec67c625699)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[00:00:06.794]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:00:03.1874189Z);\\nlet _endTime = datetime(2026-08-25T00:00:03.1874200Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6pmwg8htdhg9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-25T00:00:06.6685554Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8161fae6-5ba5-437e-9969-ded5f2fa4099\\\",\\r\\n \\\"activityId\\\": \\\"f6ea14a6-45ea-406a-84a9-e4660aa2592c\\\",\\r\\n \\\"subActivityId\\\": \\\"a15ff010-b600-4c98-b9b4-eec67c625699\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"95b62cbe-f648-4de9-8ee1-66a05ab19238\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8161fae6-5ba5-437e-9969-ded5f2fa4099 ARID=f6ea14a6-45ea-406a-84a9-e4660aa2592c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4a46171d-4922-4ebe-a2fb-926ed44edad6 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4a46171d-4922-4ebe-a2fb-926ed44edad6 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/95b62cbe-f648-4de9-8ee1-66a05ab19238 \\u003e DN.FE.ExecuteQuery/a15ff010-b600-4c98-b9b4-eec67c625699)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"b8e11d5fc111d5c1f090489c5c76193e3592cbb4c7ea0d35fc1dc1aa660c3a16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","failed_at":"alert","signature_id":"f8ca631bf4ba3de6a823751843028c53bf8f63552f781736fc0f06124e42925e","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T23:47:43Z\n Ended: 2026-08-25T00:09:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4709760-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd8f93h71j1qrcah2q03eja41gojmif-i6w1a1c6f6m8f7l\", pod=\"router-7f97cf9c5-phjp6\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd8f93h71j1qrcah2q03eja41gojmif-i6w1a1c6f6m8f7l/router-7f97cf9c5-phjp6 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092019318316339200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6663,"pr_state":"open","pr_sha":"e8abf5de3132408aecdd59b931638b0f3abb6b7a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T22:40:47Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3085bb6d6a8ff01d03a342cdde77431332338cd181de149dfacdd340ad697552","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092019318316339200","occurred_at":"2026-08-24T22:40:47Z","failed_at":"e2e","signature_id":"a6a8523a32b9fba91a90fbd04abe90465e11b86cdf06949b929a37a85e4f40ad","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d48840\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-mglhphvdtjhd\" to finish creating: GET https://rp.j6339200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\",\n \"name\": \"1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:29:18.405302537Z\",\n \"endTime\": \"2026-08-24T23:48:26.458954406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d48840\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-mglhphvdtjhd\" to finish creating: GET https://rp.j6339200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\",\n \"name\": \"1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:29:18.405302537Z\",\n \"endTime\": \"2026-08-24T23:48:26.458954406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cd8aa74f63552d51334d4e11dc816ab69810458616a41cd5ccfb369cf12b68af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092019318316339200","occurred_at":"2026-08-24T22:40:47Z","failed_at":"e2e","signature_id":"e6285c4c12e9e0c85b5ecc25058de40b9f05b37d1a08c8a8c4a66108766b8ee0","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010a6520\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ztqxhwgzpbxg\" to finish creating: GET https://rp.j6339200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8cd452b-9528-42a8-b489-fc4c1edf89ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8cd452b-9528-42a8-b489-fc4c1edf89ab\",\n \"name\": \"a8cd452b-9528-42a8-b489-fc4c1edf89ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:28:24.136815957Z\",\n \"endTime\": \"2026-08-24T23:47:26.678538436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010a6520\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ztqxhwgzpbxg\" to finish creating: GET https://rp.j6339200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8cd452b-9528-42a8-b489-fc4c1edf89ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8cd452b-9528-42a8-b489-fc4c1edf89ab\",\n \"name\": \"a8cd452b-9528-42a8-b489-fc4c1edf89ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:28:24.136815957Z\",\n \"endTime\": \"2026-08-24T23:47:26.678538436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6658,"pr_state":"open","pr_sha":"86bc57f75bd4bc7d3ab9f59b27225fa5422625d2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T22:28:33Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1293ccb55d00ab8e8cdd14d39e0e03292c0ed039cc3f1955d4324ad8e0afa07f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","failed_at":"e2e","signature_id":"916b71cbe1c0282bebb7706c18cf185456f94291c6c7059f8d673bee6920e761","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001404860\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-4vpnfc6cs6l8\" to finish creating: GET https://rp.j4721408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56cea9aa-e8d5-4c87-9934-6423e81ae186\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56cea9aa-e8d5-4c87-9934-6423e81ae186\",\n \"name\": \"56cea9aa-e8d5-4c87-9934-6423e81ae186\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:13:22.86871269Z\",\n \"endTime\": \"2026-08-24T23:32:23.316177558Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001404860\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-4vpnfc6cs6l8\" to finish creating: GET https://rp.j4721408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56cea9aa-e8d5-4c87-9934-6423e81ae186\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56cea9aa-e8d5-4c87-9934-6423e81ae186\",\n \"name\": \"56cea9aa-e8d5-4c87-9934-6423e81ae186\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:13:22.86871269Z\",\n \"endTime\": \"2026-08-24T23:32:23.316177558Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"584f10c01bb2ad702f6a246842d2b02ae338f192158b85d7050f63ba52b3b7bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","failed_at":"e2e","signature_id":"4549c28d9f346a4101b2d199c56363a0a36787b6952f32440182d5210d9903d4","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e90140\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-zmfzwkvx7j5m\" to finish creating: GET https://rp.j4721408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0f0de8f-0832-4d52-8107-854a84528fff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0f0de8f-0832-4d52-8107-854a84528fff\",\n \"name\": \"f0f0de8f-0832-4d52-8107-854a84528fff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:13:10.586040355Z\",\n \"endTime\": \"2026-08-24T23:32:13.301339783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e90140\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-zmfzwkvx7j5m\" to finish creating: GET https://rp.j4721408.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0f0de8f-0832-4d52-8107-854a84528fff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0f0de8f-0832-4d52-8107-854a84528fff\",\n \"name\": \"f0f0de8f-0832-4d52-8107-854a84528fff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:13:10.586040355Z\",\n \"endTime\": \"2026-08-24T23:32:13.301339783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6b9e188b8c18cee2cdcf45039de52d16d7502140818819b1945afb56fe8b4791","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","failed_at":"e2e","signature_id":"0d5221b00aeb7bfdca1da1581a9d7578e9d19b098899c13f87a46837f2394f22","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000213240\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[23:38:31.458]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:38:28.2685815Z);\\nlet _endTime = datetime(2026-08-24T23:38:28.2685822Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-pgv7kxdwgbs9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:38:31.4336345Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f3cb75f1-d7c9-4b87-adcc-5c7920c1ae76\\\",\\r\\n \\\"activityId\\\": \\\"13caf9ec-f590-4f8b-a99f-4bb19faac07c\\\",\\r\\n \\\"subActivityId\\\": \\\"789e5af0-c128-4f5a-8973-5ec8f7451bb0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"5a471082-4419-4989-9a9e-467c6c7c4fd6\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f3cb75f1-d7c9-4b87-adcc-5c7920c1ae76 ARID=13caf9ec-f590-4f8b-a99f-4bb19faac07c \\u003e GW.Http.CallContext/13caf9ec-f590-4f8b-a99f-4bb19faac07c \\u003e GW.ExecuteQuery/5a471082-4419-4989-9a9e-467c6c7c4fd6 \\u003e DN.FE.ExecuteQuery/789e5af0-c128-4f5a-8973-5ec8f7451bb0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:38:31.517]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:38:28.2685815Z);\\nlet _endTime = datetime(2026-08-24T23:38:28.2685822Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-pgv7kxdwgbs9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:38:31.4336345Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f3cb75f1-d7c9-4b87-adcc-5c7920c1ae76\\\",\\r\\n \\\"activityId\\\": \\\"13caf9ec-f590-4f8b-a99f-4bb19faac07c\\\",\\r\\n \\\"subActivityId\\\": \\\"789e5af0-c128-4f5a-8973-5ec8f7451bb0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"5a471082-4419-4989-9a9e-467c6c7c4fd6\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f3cb75f1-d7c9-4b87-adcc-5c7920c1ae76 ARID=13caf9ec-f590-4f8b-a99f-4bb19faac07c \\u003e GW.Http.CallContext/13caf9ec-f590-4f8b-a99f-4bb19faac07c \\u003e GW.ExecuteQuery/5a471082-4419-4989-9a9e-467c6c7c4fd6 \\u003e DN.FE.ExecuteQuery/789e5af0-c128-4f5a-8973-5ec8f7451bb0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[23:38:34.673]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:38:31.5847811Z);\\nlet _endTime = datetime(2026-08-24T23:38:31.5847819Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-pgv7kxdwgbs9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:38:34.6523649Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;defe9e59-a9e0-410e-9251-a03a8a2d2f6b\\\",\\r\\n \\\"activityId\\\": \\\"56e0ac7d-4433-4a86-9cfe-6905af57c05a\\\",\\r\\n \\\"subActivityId\\\": \\\"ad63d932-4f9e-4483-9ab8-8ec613cf4164\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6ed21c2c-224d-44ac-9721-302de763024e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;defe9e59-a9e0-410e-9251-a03a8a2d2f6b ARID=56e0ac7d-4433-4a86-9cfe-6905af57c05a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4ddc6f31-9ca8-4fa6-acc9-e2c28ed3c59d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4ddc6f31-9ca8-4fa6-acc9-e2c28ed3c59d \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6ed21c2c-224d-44ac-9721-302de763024e \\u003e DN.FE.ExecuteQuery/ad63d932-4f9e-4483-9ab8-8ec613cf4164)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:38:34.725]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:38:31.5847811Z);\\nlet _endTime = datetime(2026-08-24T23:38:31.5847819Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-pgv7kxdwgbs9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:38:34.6523649Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;defe9e59-a9e0-410e-9251-a03a8a2d2f6b\\\",\\r\\n \\\"activityId\\\": \\\"56e0ac7d-4433-4a86-9cfe-6905af57c05a\\\",\\r\\n \\\"subActivityId\\\": \\\"ad63d932-4f9e-4483-9ab8-8ec613cf4164\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6ed21c2c-224d-44ac-9721-302de763024e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;defe9e59-a9e0-410e-9251-a03a8a2d2f6b ARID=56e0ac7d-4433-4a86-9cfe-6905af57c05a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4ddc6f31-9ca8-4fa6-acc9-e2c28ed3c59d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4ddc6f31-9ca8-4fa6-acc9-e2c28ed3c59d \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6ed21c2c-224d-44ac-9721-302de763024e \\u003e DN.FE.ExecuteQuery/ad63d932-4f9e-4483-9ab8-8ec613cf4164)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[23:38:51.573]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:38:48.1146150Z);\\nlet _endTime = datetime(2026-08-24T23:38:48.1146157Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-pgv7kxdwgbs9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:38:51.5509925Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;54a98755-cda5-4e00-b75a-8a9f4a05e23a\\\",\\r\\n \\\"activityId\\\": \\\"07bd916f-8e74-4f33-9544-38554323dccf\\\",\\r\\n \\\"subActivityId\\\": \\\"ee30f34f-230c-4bd2-af26-3c459201c20c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"95379035-872f-45ec-8ef7-b6611ac9f3a9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54a98755-cda5-4e00-b75a-8a9f4a05e23a ARID=07bd916f-8e74-4f33-9544-38554323dccf \\u003e GW.Http.CallContext/07bd916f-8e74-4f33-9544-38554323dccf \\u003e GW.ExecuteQuery/95379035-872f-45ec-8ef7-b6611ac9f3a9 \\u003e DN.FE.ExecuteQuery/ee30f34f-230c-4bd2-af26-3c459201c20c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:38:51.682]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:38:48.1146150Z);\\nlet _endTime = datetime(2026-08-24T23:38:48.1146157Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-pgv7kxdwgbs9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:38:51.5509925Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;54a98755-cda5-4e00-b75a-8a9f4a05e23a\\\",\\r\\n \\\"activityId\\\": \\\"07bd916f-8e74-4f33-9544-38554323dccf\\\",\\r\\n \\\"subActivityId\\\": \\\"ee30f34f-230c-4bd2-af26-3c459201c20c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"95379035-872f-45ec-8ef7-b6611ac9f3a9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54a98755-cda5-4e00-b75a-8a9f4a05e23a ARID=07bd916f-8e74-4f33-9544-38554323dccf \\u003e GW.Http.CallContext/07bd916f-8e74-4f33-9544-38554323dccf \\u003e GW.ExecuteQuery/95379035-872f-45ec-8ef7-b6611ac9f3a9 \\u003e DN.FE.ExecuteQuery/ee30f34f-230c-4bd2-af26-3c459201c20c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000213240\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[23:38:31.458]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:38:28.2685815Z);\\nlet _endTime = datetime(2026-08-24T23:38:28.2685822Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-pgv7kxdwgbs9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:38:31.4336345Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f3cb75f1-d7c9-4b87-adcc-5c7920c1ae76\\\",\\r\\n \\\"activityId\\\": \\\"13caf9ec-f590-4f8b-a99f-4bb19faac07c\\\",\\r\\n \\\"subActivityId\\\": \\\"789e5af0-c128-4f5a-8973-5ec8f7451bb0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"5a471082-4419-4989-9a9e-467c6c7c4fd6\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f3cb75f1-d7c9-4b87-adcc-5c7920c1ae76 ARID=13caf9ec-f590-4f8b-a99f-4bb19faac07c \\u003e GW.Http.CallContext/13caf9ec-f590-4f8b-a99f-4bb19faac07c \\u003e GW.ExecuteQuery/5a471082-4419-4989-9a9e-467c6c7c4fd6 \\u003e DN.FE.ExecuteQuery/789e5af0-c128-4f5a-8973-5ec8f7451bb0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:38:31.517]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:38:28.2685815Z);\\nlet _endTime = datetime(2026-08-24T23:38:28.2685822Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-pgv7kxdwgbs9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:38:31.4336345Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f3cb75f1-d7c9-4b87-adcc-5c7920c1ae76\\\",\\r\\n \\\"activityId\\\": \\\"13caf9ec-f590-4f8b-a99f-4bb19faac07c\\\",\\r\\n \\\"subActivityId\\\": \\\"789e5af0-c128-4f5a-8973-5ec8f7451bb0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"5a471082-4419-4989-9a9e-467c6c7c4fd6\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f3cb75f1-d7c9-4b87-adcc-5c7920c1ae76 ARID=13caf9ec-f590-4f8b-a99f-4bb19faac07c \\u003e GW.Http.CallContext/13caf9ec-f590-4f8b-a99f-4bb19faac07c \\u003e GW.ExecuteQuery/5a471082-4419-4989-9a9e-467c6c7c4fd6 \\u003e DN.FE.ExecuteQuery/789e5af0-c128-4f5a-8973-5ec8f7451bb0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[23:38:34.673]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:38:31.5847811Z);\\nlet _endTime = datetime(2026-08-24T23:38:31.5847819Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-pgv7kxdwgbs9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:38:34.6523649Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;defe9e59-a9e0-410e-9251-a03a8a2d2f6b\\\",\\r\\n \\\"activityId\\\": \\\"56e0ac7d-4433-4a86-9cfe-6905af57c05a\\\",\\r\\n \\\"subActivityId\\\": \\\"ad63d932-4f9e-4483-9ab8-8ec613cf4164\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6ed21c2c-224d-44ac-9721-302de763024e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;defe9e59-a9e0-410e-9251-a03a8a2d2f6b ARID=56e0ac7d-4433-4a86-9cfe-6905af57c05a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4ddc6f31-9ca8-4fa6-acc9-e2c28ed3c59d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4ddc6f31-9ca8-4fa6-acc9-e2c28ed3c59d \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6ed21c2c-224d-44ac-9721-302de763024e \\u003e DN.FE.ExecuteQuery/ad63d932-4f9e-4483-9ab8-8ec613cf4164)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:38:34.725]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:38:31.5847811Z);\\nlet _endTime = datetime(2026-08-24T23:38:31.5847819Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-pgv7kxdwgbs9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:38:34.6523649Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;defe9e59-a9e0-410e-9251-a03a8a2d2f6b\\\",\\r\\n \\\"activityId\\\": \\\"56e0ac7d-4433-4a86-9cfe-6905af57c05a\\\",\\r\\n \\\"subActivityId\\\": \\\"ad63d932-4f9e-4483-9ab8-8ec613cf4164\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6ed21c2c-224d-44ac-9721-302de763024e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;defe9e59-a9e0-410e-9251-a03a8a2d2f6b ARID=56e0ac7d-4433-4a86-9cfe-6905af57c05a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4ddc6f31-9ca8-4fa6-acc9-e2c28ed3c59d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/4ddc6f31-9ca8-4fa6-acc9-e2c28ed3c59d \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6ed21c2c-224d-44ac-9721-302de763024e \\u003e DN.FE.ExecuteQuery/ad63d932-4f9e-4483-9ab8-8ec613cf4164)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[23:38:51.573]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:38:48.1146150Z);\\nlet _endTime = datetime(2026-08-24T23:38:48.1146157Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-pgv7kxdwgbs9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:38:51.5509925Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;54a98755-cda5-4e00-b75a-8a9f4a05e23a\\\",\\r\\n \\\"activityId\\\": \\\"07bd916f-8e74-4f33-9544-38554323dccf\\\",\\r\\n \\\"subActivityId\\\": \\\"ee30f34f-230c-4bd2-af26-3c459201c20c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"95379035-872f-45ec-8ef7-b6611ac9f3a9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54a98755-cda5-4e00-b75a-8a9f4a05e23a ARID=07bd916f-8e74-4f33-9544-38554323dccf \\u003e GW.Http.CallContext/07bd916f-8e74-4f33-9544-38554323dccf \\u003e GW.ExecuteQuery/95379035-872f-45ec-8ef7-b6611ac9f3a9 \\u003e DN.FE.ExecuteQuery/ee30f34f-230c-4bd2-af26-3c459201c20c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:38:51.682]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:38:48.1146150Z);\\nlet _endTime = datetime(2026-08-24T23:38:48.1146157Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-pgv7kxdwgbs9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:38:51.5509925Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;54a98755-cda5-4e00-b75a-8a9f4a05e23a\\\",\\r\\n \\\"activityId\\\": \\\"07bd916f-8e74-4f33-9544-38554323dccf\\\",\\r\\n \\\"subActivityId\\\": \\\"ee30f34f-230c-4bd2-af26-3c459201c20c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"95379035-872f-45ec-8ef7-b6611ac9f3a9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;54a98755-cda5-4e00-b75a-8a9f4a05e23a ARID=07bd916f-8e74-4f33-9544-38554323dccf \\u003e GW.Http.CallContext/07bd916f-8e74-4f33-9544-38554323dccf \\u003e GW.ExecuteQuery/95379035-872f-45ec-8ef7-b6611ac9f3a9 \\u003e DN.FE.ExecuteQuery/ee30f34f-230c-4bd2-af26-3c459201c20c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092014812023230464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6662,"pr_state":"open","pr_sha":"87c9d3ce24afa32ec6516cc621a3203652172097","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T22:22:53Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"047d969656b19b343427887358269ed4cadde30dbc61c55e456f543e143b538c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092014812023230464","occurred_at":"2026-08-24T22:22:53Z","failed_at":"e2e","signature_id":"c5fd89f1c41b331bfa0c3c5ff918b5ee94282a73960beb6c8680dfabd0d92aba","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001f02a40\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[23:17:20.322]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:17:17.0513987Z);\\nlet _endTime = datetime(2026-08-24T23:17:17.0513993Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ss9mztkxcbtx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:17:20.3014069Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;bdf5bb75-f3a4-4922-b252-582f3654b42a\\\",\\r\\n \\\"activityId\\\": \\\"5d98f621-5732-4f3e-99d2-21c3ecc8195a\\\",\\r\\n \\\"subActivityId\\\": \\\"7a976fd8-5470-4ca6-9a6a-3e041417cc3c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bb8e1bae-a54b-4f1b-aa1a-86b94820a3f9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bdf5bb75-f3a4-4922-b252-582f3654b42a ARID=5d98f621-5732-4f3e-99d2-21c3ecc8195a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2fb79466-7c27-42f3-a6c2-866bddfdc399 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2fb79466-7c27-42f3-a6c2-866bddfdc399 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/bb8e1bae-a54b-4f1b-aa1a-86b94820a3f9 \\u003e DN.FE.ExecuteQuery/7a976fd8-5470-4ca6-9a6a-3e041417cc3c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:17:20.389]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:17:17.0513987Z);\\nlet _endTime = datetime(2026-08-24T23:17:17.0513993Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ss9mztkxcbtx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:17:20.3014069Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;bdf5bb75-f3a4-4922-b252-582f3654b42a\\\",\\r\\n \\\"activityId\\\": \\\"5d98f621-5732-4f3e-99d2-21c3ecc8195a\\\",\\r\\n \\\"subActivityId\\\": \\\"7a976fd8-5470-4ca6-9a6a-3e041417cc3c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bb8e1bae-a54b-4f1b-aa1a-86b94820a3f9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bdf5bb75-f3a4-4922-b252-582f3654b42a ARID=5d98f621-5732-4f3e-99d2-21c3ecc8195a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2fb79466-7c27-42f3-a6c2-866bddfdc399 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2fb79466-7c27-42f3-a6c2-866bddfdc399 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/bb8e1bae-a54b-4f1b-aa1a-86b94820a3f9 \\u003e DN.FE.ExecuteQuery/7a976fd8-5470-4ca6-9a6a-3e041417cc3c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[23:17:23.477]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:17:20.4650185Z);\\nlet _endTime = datetime(2026-08-24T23:17:20.4650189Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ss9mztkxcbtx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:17:23.4523754Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2e051b49-6e32-48db-a742-06c75efc4f02\\\",\\r\\n \\\"activityId\\\": \\\"dfb8edfc-26ab-4141-b358-423d3a40e76b\\\",\\r\\n \\\"subActivityId\\\": \\\"a58cffec-2cfe-4210-bf95-cf8a49e5d515\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d7e2f919-501a-4524-8f3e-7eadaba81ac8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2e051b49-6e32-48db-a742-06c75efc4f02 ARID=dfb8edfc-26ab-4141-b358-423d3a40e76b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7364b8c6-dd3b-462e-a8cd-fdd5d2ebf551 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7364b8c6-dd3b-462e-a8cd-fdd5d2ebf551 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/d7e2f919-501a-4524-8f3e-7eadaba81ac8 \\u003e DN.FE.ExecuteQuery/a58cffec-2cfe-4210-bf95-cf8a49e5d515)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:17:23.548]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:17:20.4650185Z);\\nlet _endTime = datetime(2026-08-24T23:17:20.4650189Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ss9mztkxcbtx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:17:23.4523754Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2e051b49-6e32-48db-a742-06c75efc4f02\\\",\\r\\n \\\"activityId\\\": \\\"dfb8edfc-26ab-4141-b358-423d3a40e76b\\\",\\r\\n \\\"subActivityId\\\": \\\"a58cffec-2cfe-4210-bf95-cf8a49e5d515\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d7e2f919-501a-4524-8f3e-7eadaba81ac8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2e051b49-6e32-48db-a742-06c75efc4f02 ARID=dfb8edfc-26ab-4141-b358-423d3a40e76b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7364b8c6-dd3b-462e-a8cd-fdd5d2ebf551 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7364b8c6-dd3b-462e-a8cd-fdd5d2ebf551 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/d7e2f919-501a-4524-8f3e-7eadaba81ac8 \\u003e DN.FE.ExecuteQuery/a58cffec-2cfe-4210-bf95-cf8a49e5d515)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[23:17:30.062]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:17:26.7994218Z);\\nlet _endTime = datetime(2026-08-24T23:17:26.7994224Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ss9mztkxcbtx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:17:30.0404281Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a928551c-c326-4e69-837c-028a41ddae49\\\",\\r\\n \\\"activityId\\\": \\\"8a5d47f3-bd6e-4004-afb3-f9fe45c95ae5\\\",\\r\\n \\\"subActivityId\\\": \\\"7711a873-92d6-4318-afed-c5650fa8c291\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"19e8c6db-27ba-4efc-ab96-cbc1459b225d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a928551c-c326-4e69-837c-028a41ddae49 ARID=8a5d47f3-bd6e-4004-afb3-f9fe45c95ae5 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8a7841e7-91c1-4a08-81b4-539f76ffd5b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8a7841e7-91c1-4a08-81b4-539f76ffd5b7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/19e8c6db-27ba-4efc-ab96-cbc1459b225d \\u003e DN.FE.ExecuteQuery/7711a873-92d6-4318-afed-c5650fa8c291)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:17:30.188]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:17:26.7994218Z);\\nlet _endTime = datetime(2026-08-24T23:17:26.7994224Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ss9mztkxcbtx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:17:30.0404281Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a928551c-c326-4e69-837c-028a41ddae49\\\",\\r\\n \\\"activityId\\\": \\\"8a5d47f3-bd6e-4004-afb3-f9fe45c95ae5\\\",\\r\\n \\\"subActivityId\\\": \\\"7711a873-92d6-4318-afed-c5650fa8c291\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"19e8c6db-27ba-4efc-ab96-cbc1459b225d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a928551c-c326-4e69-837c-028a41ddae49 ARID=8a5d47f3-bd6e-4004-afb3-f9fe45c95ae5 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8a7841e7-91c1-4a08-81b4-539f76ffd5b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8a7841e7-91c1-4a08-81b4-539f76ffd5b7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/19e8c6db-27ba-4efc-ab96-cbc1459b225d \\u003e DN.FE.ExecuteQuery/7711a873-92d6-4318-afed-c5650fa8c291)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001f02a40\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[23:17:20.322]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:17:17.0513987Z);\\nlet _endTime = datetime(2026-08-24T23:17:17.0513993Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ss9mztkxcbtx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:17:20.3014069Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;bdf5bb75-f3a4-4922-b252-582f3654b42a\\\",\\r\\n \\\"activityId\\\": \\\"5d98f621-5732-4f3e-99d2-21c3ecc8195a\\\",\\r\\n \\\"subActivityId\\\": \\\"7a976fd8-5470-4ca6-9a6a-3e041417cc3c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bb8e1bae-a54b-4f1b-aa1a-86b94820a3f9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bdf5bb75-f3a4-4922-b252-582f3654b42a ARID=5d98f621-5732-4f3e-99d2-21c3ecc8195a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2fb79466-7c27-42f3-a6c2-866bddfdc399 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2fb79466-7c27-42f3-a6c2-866bddfdc399 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/bb8e1bae-a54b-4f1b-aa1a-86b94820a3f9 \\u003e DN.FE.ExecuteQuery/7a976fd8-5470-4ca6-9a6a-3e041417cc3c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:17:20.389]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:17:17.0513987Z);\\nlet _endTime = datetime(2026-08-24T23:17:17.0513993Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ss9mztkxcbtx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:17:20.3014069Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;bdf5bb75-f3a4-4922-b252-582f3654b42a\\\",\\r\\n \\\"activityId\\\": \\\"5d98f621-5732-4f3e-99d2-21c3ecc8195a\\\",\\r\\n \\\"subActivityId\\\": \\\"7a976fd8-5470-4ca6-9a6a-3e041417cc3c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bb8e1bae-a54b-4f1b-aa1a-86b94820a3f9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;bdf5bb75-f3a4-4922-b252-582f3654b42a ARID=5d98f621-5732-4f3e-99d2-21c3ecc8195a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2fb79466-7c27-42f3-a6c2-866bddfdc399 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2fb79466-7c27-42f3-a6c2-866bddfdc399 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/bb8e1bae-a54b-4f1b-aa1a-86b94820a3f9 \\u003e DN.FE.ExecuteQuery/7a976fd8-5470-4ca6-9a6a-3e041417cc3c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[23:17:23.477]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:17:20.4650185Z);\\nlet _endTime = datetime(2026-08-24T23:17:20.4650189Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ss9mztkxcbtx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:17:23.4523754Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2e051b49-6e32-48db-a742-06c75efc4f02\\\",\\r\\n \\\"activityId\\\": \\\"dfb8edfc-26ab-4141-b358-423d3a40e76b\\\",\\r\\n \\\"subActivityId\\\": \\\"a58cffec-2cfe-4210-bf95-cf8a49e5d515\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d7e2f919-501a-4524-8f3e-7eadaba81ac8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2e051b49-6e32-48db-a742-06c75efc4f02 ARID=dfb8edfc-26ab-4141-b358-423d3a40e76b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7364b8c6-dd3b-462e-a8cd-fdd5d2ebf551 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7364b8c6-dd3b-462e-a8cd-fdd5d2ebf551 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/d7e2f919-501a-4524-8f3e-7eadaba81ac8 \\u003e DN.FE.ExecuteQuery/a58cffec-2cfe-4210-bf95-cf8a49e5d515)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:17:23.548]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:17:20.4650185Z);\\nlet _endTime = datetime(2026-08-24T23:17:20.4650189Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ss9mztkxcbtx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:17:23.4523754Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2e051b49-6e32-48db-a742-06c75efc4f02\\\",\\r\\n \\\"activityId\\\": \\\"dfb8edfc-26ab-4141-b358-423d3a40e76b\\\",\\r\\n \\\"subActivityId\\\": \\\"a58cffec-2cfe-4210-bf95-cf8a49e5d515\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d7e2f919-501a-4524-8f3e-7eadaba81ac8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2e051b49-6e32-48db-a742-06c75efc4f02 ARID=dfb8edfc-26ab-4141-b358-423d3a40e76b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7364b8c6-dd3b-462e-a8cd-fdd5d2ebf551 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7364b8c6-dd3b-462e-a8cd-fdd5d2ebf551 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/d7e2f919-501a-4524-8f3e-7eadaba81ac8 \\u003e DN.FE.ExecuteQuery/a58cffec-2cfe-4210-bf95-cf8a49e5d515)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[23:17:30.062]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:17:26.7994218Z);\\nlet _endTime = datetime(2026-08-24T23:17:26.7994224Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ss9mztkxcbtx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:17:30.0404281Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a928551c-c326-4e69-837c-028a41ddae49\\\",\\r\\n \\\"activityId\\\": \\\"8a5d47f3-bd6e-4004-afb3-f9fe45c95ae5\\\",\\r\\n \\\"subActivityId\\\": \\\"7711a873-92d6-4318-afed-c5650fa8c291\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"19e8c6db-27ba-4efc-ab96-cbc1459b225d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a928551c-c326-4e69-837c-028a41ddae49 ARID=8a5d47f3-bd6e-4004-afb3-f9fe45c95ae5 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8a7841e7-91c1-4a08-81b4-539f76ffd5b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8a7841e7-91c1-4a08-81b4-539f76ffd5b7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/19e8c6db-27ba-4efc-ab96-cbc1459b225d \\u003e DN.FE.ExecuteQuery/7711a873-92d6-4318-afed-c5650fa8c291)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:17:30.188]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T23:17:26.7994218Z);\\nlet _endTime = datetime(2026-08-24T23:17:26.7994224Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ss9mztkxcbtx'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T23:17:30.0404281Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a928551c-c326-4e69-837c-028a41ddae49\\\",\\r\\n \\\"activityId\\\": \\\"8a5d47f3-bd6e-4004-afb3-f9fe45c95ae5\\\",\\r\\n \\\"subActivityId\\\": \\\"7711a873-92d6-4318-afed-c5650fa8c291\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"19e8c6db-27ba-4efc-ab96-cbc1459b225d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a928551c-c326-4e69-837c-028a41ddae49 ARID=8a5d47f3-bd6e-4004-afb3-f9fe45c95ae5 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8a7841e7-91c1-4a08-81b4-539f76ffd5b7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8a7841e7-91c1-4a08-81b4-539f76ffd5b7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/19e8c6db-27ba-4efc-ab96-cbc1459b225d \\u003e DN.FE.ExecuteQuery/7711a873-92d6-4318-afed-c5650fa8c291)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092003732647579648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6088,"pr_state":"open","pr_sha":"5fa9bda94a960560b9f4d097c9466ff24895e7ba","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T21:38:47Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bfb82e69732ddc9bb82e12e65b76679fd8ea4fc7803581c4847ef806415611e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092003732647579648","occurred_at":"2026-08-24T21:38:47Z","failed_at":"e2e","signature_id":"c3920cca5c3b22e9ea0a8df63d7fd82dff370a2c49fd2a56d2b8612e6611e8cd","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000d24a80\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[22:49:59.230]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:49:56.2562204Z);\\nlet _endTime = datetime(2026-08-24T22:49:56.2562208Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-gqhhcj25cdch'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:49:59.2093178Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c3bef575-02f0-4e36-bf01-e442f3b91c12\\\",\\r\\n \\\"activityId\\\": \\\"6ed899e3-526b-485a-86d9-7f14c7afb204\\\",\\r\\n \\\"subActivityId\\\": \\\"9aef40a3-6517-4779-af80-2a394ccc34f0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0310a9c3-e142-486e-8e89-ea1972124e7e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c3bef575-02f0-4e36-bf01-e442f3b91c12 ARID=6ed899e3-526b-485a-86d9-7f14c7afb204 \\u003e GW.Http.CallContext/6ed899e3-526b-485a-86d9-7f14c7afb204 \\u003e GW.ExecuteQuery/0310a9c3-e142-486e-8e89-ea1972124e7e \\u003e DN.FE.ExecuteQuery/9aef40a3-6517-4779-af80-2a394ccc34f0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:49:59.276]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:49:56.2562204Z);\\nlet _endTime = datetime(2026-08-24T22:49:56.2562208Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-gqhhcj25cdch'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:49:59.2093178Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c3bef575-02f0-4e36-bf01-e442f3b91c12\\\",\\r\\n \\\"activityId\\\": \\\"6ed899e3-526b-485a-86d9-7f14c7afb204\\\",\\r\\n \\\"subActivityId\\\": \\\"9aef40a3-6517-4779-af80-2a394ccc34f0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0310a9c3-e142-486e-8e89-ea1972124e7e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c3bef575-02f0-4e36-bf01-e442f3b91c12 ARID=6ed899e3-526b-485a-86d9-7f14c7afb204 \\u003e GW.Http.CallContext/6ed899e3-526b-485a-86d9-7f14c7afb204 \\u003e GW.ExecuteQuery/0310a9c3-e142-486e-8e89-ea1972124e7e \\u003e DN.FE.ExecuteQuery/9aef40a3-6517-4779-af80-2a394ccc34f0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[22:50:02.572]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:49:59.3623198Z);\\nlet _endTime = datetime(2026-08-24T22:49:59.3623202Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-gqhhcj25cdch'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:50:02.5524012Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;338abe06-3616-4676-bb86-93200535d787\\\",\\r\\n \\\"activityId\\\": \\\"d9ff4261-86d5-403c-864a-c18c8e8cdd4b\\\",\\r\\n \\\"subActivityId\\\": \\\"f710bc94-d353-4e91-a20d-461bdfd636c9\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"60453caa-4266-46ca-b70c-374022fcf35e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;338abe06-3616-4676-bb86-93200535d787 ARID=d9ff4261-86d5-403c-864a-c18c8e8cdd4b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/29e8a84d-1cb3-4fe7-ab1a-0e3f4cdbb700 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/29e8a84d-1cb3-4fe7-ab1a-0e3f4cdbb700 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/60453caa-4266-46ca-b70c-374022fcf35e \\u003e DN.FE.ExecuteQuery/f710bc94-d353-4e91-a20d-461bdfd636c9)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:50:02.623]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:49:59.3623198Z);\\nlet _endTime = datetime(2026-08-24T22:49:59.3623202Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-gqhhcj25cdch'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:50:02.5524012Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;338abe06-3616-4676-bb86-93200535d787\\\",\\r\\n \\\"activityId\\\": \\\"d9ff4261-86d5-403c-864a-c18c8e8cdd4b\\\",\\r\\n \\\"subActivityId\\\": \\\"f710bc94-d353-4e91-a20d-461bdfd636c9\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"60453caa-4266-46ca-b70c-374022fcf35e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;338abe06-3616-4676-bb86-93200535d787 ARID=d9ff4261-86d5-403c-864a-c18c8e8cdd4b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/29e8a84d-1cb3-4fe7-ab1a-0e3f4cdbb700 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/29e8a84d-1cb3-4fe7-ab1a-0e3f4cdbb700 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/60453caa-4266-46ca-b70c-374022fcf35e \\u003e DN.FE.ExecuteQuery/f710bc94-d353-4e91-a20d-461bdfd636c9)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[22:50:09.721]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:50:06.2998581Z);\\nlet _endTime = datetime(2026-08-24T22:50:06.2998587Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-gqhhcj25cdch'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:50:09.7003481Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ec1905a7-7ca7-43e4-9299-c9fb7166dc78\\\",\\r\\n \\\"activityId\\\": \\\"2c4c24ce-6ca1-4092-b20f-62aa7ce11dac\\\",\\r\\n \\\"subActivityId\\\": \\\"828b6fa7-bf28-4ff4-a26e-c89a97292ff3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ca11ea7c-ce7e-4449-bfe4-feb2c1797062\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ec1905a7-7ca7-43e4-9299-c9fb7166dc78 ARID=2c4c24ce-6ca1-4092-b20f-62aa7ce11dac \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/57e05fdd-42df-49c9-9b68-aca74f530e51 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/57e05fdd-42df-49c9-9b68-aca74f530e51 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ca11ea7c-ce7e-4449-bfe4-feb2c1797062 \\u003e DN.FE.ExecuteQuery/828b6fa7-bf28-4ff4-a26e-c89a97292ff3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:50:09.823]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:50:06.2998581Z);\\nlet _endTime = datetime(2026-08-24T22:50:06.2998587Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-gqhhcj25cdch'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:50:09.7003481Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ec1905a7-7ca7-43e4-9299-c9fb7166dc78\\\",\\r\\n \\\"activityId\\\": \\\"2c4c24ce-6ca1-4092-b20f-62aa7ce11dac\\\",\\r\\n \\\"subActivityId\\\": \\\"828b6fa7-bf28-4ff4-a26e-c89a97292ff3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ca11ea7c-ce7e-4449-bfe4-feb2c1797062\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ec1905a7-7ca7-43e4-9299-c9fb7166dc78 ARID=2c4c24ce-6ca1-4092-b20f-62aa7ce11dac \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/57e05fdd-42df-49c9-9b68-aca74f530e51 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/57e05fdd-42df-49c9-9b68-aca74f530e51 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ca11ea7c-ce7e-4449-bfe4-feb2c1797062 \\u003e DN.FE.ExecuteQuery/828b6fa7-bf28-4ff4-a26e-c89a97292ff3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000d24a80\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[22:49:59.230]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:49:56.2562204Z);\\nlet _endTime = datetime(2026-08-24T22:49:56.2562208Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-gqhhcj25cdch'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:49:59.2093178Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c3bef575-02f0-4e36-bf01-e442f3b91c12\\\",\\r\\n \\\"activityId\\\": \\\"6ed899e3-526b-485a-86d9-7f14c7afb204\\\",\\r\\n \\\"subActivityId\\\": \\\"9aef40a3-6517-4779-af80-2a394ccc34f0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0310a9c3-e142-486e-8e89-ea1972124e7e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c3bef575-02f0-4e36-bf01-e442f3b91c12 ARID=6ed899e3-526b-485a-86d9-7f14c7afb204 \\u003e GW.Http.CallContext/6ed899e3-526b-485a-86d9-7f14c7afb204 \\u003e GW.ExecuteQuery/0310a9c3-e142-486e-8e89-ea1972124e7e \\u003e DN.FE.ExecuteQuery/9aef40a3-6517-4779-af80-2a394ccc34f0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:49:59.276]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:49:56.2562204Z);\\nlet _endTime = datetime(2026-08-24T22:49:56.2562208Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-gqhhcj25cdch'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:49:59.2093178Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c3bef575-02f0-4e36-bf01-e442f3b91c12\\\",\\r\\n \\\"activityId\\\": \\\"6ed899e3-526b-485a-86d9-7f14c7afb204\\\",\\r\\n \\\"subActivityId\\\": \\\"9aef40a3-6517-4779-af80-2a394ccc34f0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0310a9c3-e142-486e-8e89-ea1972124e7e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c3bef575-02f0-4e36-bf01-e442f3b91c12 ARID=6ed899e3-526b-485a-86d9-7f14c7afb204 \\u003e GW.Http.CallContext/6ed899e3-526b-485a-86d9-7f14c7afb204 \\u003e GW.ExecuteQuery/0310a9c3-e142-486e-8e89-ea1972124e7e \\u003e DN.FE.ExecuteQuery/9aef40a3-6517-4779-af80-2a394ccc34f0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[22:50:02.572]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:49:59.3623198Z);\\nlet _endTime = datetime(2026-08-24T22:49:59.3623202Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-gqhhcj25cdch'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:50:02.5524012Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;338abe06-3616-4676-bb86-93200535d787\\\",\\r\\n \\\"activityId\\\": \\\"d9ff4261-86d5-403c-864a-c18c8e8cdd4b\\\",\\r\\n \\\"subActivityId\\\": \\\"f710bc94-d353-4e91-a20d-461bdfd636c9\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"60453caa-4266-46ca-b70c-374022fcf35e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;338abe06-3616-4676-bb86-93200535d787 ARID=d9ff4261-86d5-403c-864a-c18c8e8cdd4b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/29e8a84d-1cb3-4fe7-ab1a-0e3f4cdbb700 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/29e8a84d-1cb3-4fe7-ab1a-0e3f4cdbb700 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/60453caa-4266-46ca-b70c-374022fcf35e \\u003e DN.FE.ExecuteQuery/f710bc94-d353-4e91-a20d-461bdfd636c9)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:50:02.623]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:49:59.3623198Z);\\nlet _endTime = datetime(2026-08-24T22:49:59.3623202Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-gqhhcj25cdch'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:50:02.5524012Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;338abe06-3616-4676-bb86-93200535d787\\\",\\r\\n \\\"activityId\\\": \\\"d9ff4261-86d5-403c-864a-c18c8e8cdd4b\\\",\\r\\n \\\"subActivityId\\\": \\\"f710bc94-d353-4e91-a20d-461bdfd636c9\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"60453caa-4266-46ca-b70c-374022fcf35e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;338abe06-3616-4676-bb86-93200535d787 ARID=d9ff4261-86d5-403c-864a-c18c8e8cdd4b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/29e8a84d-1cb3-4fe7-ab1a-0e3f4cdbb700 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/29e8a84d-1cb3-4fe7-ab1a-0e3f4cdbb700 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/60453caa-4266-46ca-b70c-374022fcf35e \\u003e DN.FE.ExecuteQuery/f710bc94-d353-4e91-a20d-461bdfd636c9)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[22:50:09.721]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:50:06.2998581Z);\\nlet _endTime = datetime(2026-08-24T22:50:06.2998587Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-gqhhcj25cdch'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:50:09.7003481Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ec1905a7-7ca7-43e4-9299-c9fb7166dc78\\\",\\r\\n \\\"activityId\\\": \\\"2c4c24ce-6ca1-4092-b20f-62aa7ce11dac\\\",\\r\\n \\\"subActivityId\\\": \\\"828b6fa7-bf28-4ff4-a26e-c89a97292ff3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ca11ea7c-ce7e-4449-bfe4-feb2c1797062\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ec1905a7-7ca7-43e4-9299-c9fb7166dc78 ARID=2c4c24ce-6ca1-4092-b20f-62aa7ce11dac \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/57e05fdd-42df-49c9-9b68-aca74f530e51 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/57e05fdd-42df-49c9-9b68-aca74f530e51 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ca11ea7c-ce7e-4449-bfe4-feb2c1797062 \\u003e DN.FE.ExecuteQuery/828b6fa7-bf28-4ff4-a26e-c89a97292ff3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:50:09.823]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:50:06.2998581Z);\\nlet _endTime = datetime(2026-08-24T22:50:06.2998587Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-gqhhcj25cdch'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:50:09.7003481Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ec1905a7-7ca7-43e4-9299-c9fb7166dc78\\\",\\r\\n \\\"activityId\\\": \\\"2c4c24ce-6ca1-4092-b20f-62aa7ce11dac\\\",\\r\\n \\\"subActivityId\\\": \\\"828b6fa7-bf28-4ff4-a26e-c89a97292ff3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ca11ea7c-ce7e-4449-bfe4-feb2c1797062\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ec1905a7-7ca7-43e4-9299-c9fb7166dc78 ARID=2c4c24ce-6ca1-4092-b20f-62aa7ce11dac \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/57e05fdd-42df-49c9-9b68-aca74f530e51 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/57e05fdd-42df-49c9-9b68-aca74f530e51 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ca11ea7c-ce7e-4449-bfe4-feb2c1797062 \\u003e DN.FE.ExecuteQuery/828b6fa7-bf28-4ff4-a26e-c89a97292ff3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091996785936437248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6128,"pr_state":"open","pr_sha":"170f76835cc1a848764bf285f1e459b293c9fc67","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T21:11:15Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"233d3a9f20f2bb092a6565f2d7a1461639a48ca636fda1c3f52d9308931fb207","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091996785936437248","occurred_at":"2026-08-24T21:11:15Z","failed_at":"e2e","signature_id":"c79de5565353120aa973caf9da2e4e0043d28fd25ac56a7cd204827ac9e612e5","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 618.943s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000aba840\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[22:23:34.777]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:23:31.1340475Z);\\nlet _endTime = datetime(2026-08-24T22:23:31.1340479Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrgztftd846p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:23:34.7544299Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;29db54ce-825b-4bb8-9a70-02b555f68f68\\\",\\r\\n \\\"activityId\\\": \\\"64a8c8f1-b43e-49c5-b452-340f5dfc4e2c\\\",\\r\\n \\\"subActivityId\\\": \\\"705e5e56-0d99-4165-83da-d9db2f1f98e3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c1332563-9620-4ce8-b789-424edc8d3469\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;29db54ce-825b-4bb8-9a70-02b555f68f68 ARID=64a8c8f1-b43e-49c5-b452-340f5dfc4e2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/fe5bfcde-d8a3-480b-8a59-62f13d3daed7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/fe5bfcde-d8a3-480b-8a59-62f13d3daed7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/c1332563-9620-4ce8-b789-424edc8d3469 \\u003e DN.FE.ExecuteQuery/705e5e56-0d99-4165-83da-d9db2f1f98e3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:23:34.823]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:23:31.1340475Z);\\nlet _endTime = datetime(2026-08-24T22:23:31.1340479Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrgztftd846p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:23:34.7544299Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;29db54ce-825b-4bb8-9a70-02b555f68f68\\\",\\r\\n \\\"activityId\\\": \\\"64a8c8f1-b43e-49c5-b452-340f5dfc4e2c\\\",\\r\\n \\\"subActivityId\\\": \\\"705e5e56-0d99-4165-83da-d9db2f1f98e3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c1332563-9620-4ce8-b789-424edc8d3469\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;29db54ce-825b-4bb8-9a70-02b555f68f68 ARID=64a8c8f1-b43e-49c5-b452-340f5dfc4e2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/fe5bfcde-d8a3-480b-8a59-62f13d3daed7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/fe5bfcde-d8a3-480b-8a59-62f13d3daed7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/c1332563-9620-4ce8-b789-424edc8d3469 \\u003e DN.FE.ExecuteQuery/705e5e56-0d99-4165-83da-d9db2f1f98e3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[22:23:38.026]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:23:34.8927473Z);\\nlet _endTime = datetime(2026-08-24T22:23:34.8927477Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrgztftd846p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:23:38.0025191Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;025d76c9-ff35-453c-a1d4-d6e1aa8c5ae1\\\",\\r\\n \\\"activityId\\\": \\\"ef32b257-f29d-4b0c-81be-6fd6150546f1\\\",\\r\\n \\\"subActivityId\\\": \\\"efd6fb31-137b-41e3-b32d-438594e51a18\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bbf6b235-380f-4084-8005-856fcc5a3fc6\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;025d76c9-ff35-453c-a1d4-d6e1aa8c5ae1 ARID=ef32b257-f29d-4b0c-81be-6fd6150546f1 \\u003e GW.Http.CallContext/ef32b257-f29d-4b0c-81be-6fd6150546f1 \\u003e GW.ExecuteQuery/bbf6b235-380f-4084-8005-856fcc5a3fc6 \\u003e DN.FE.ExecuteQuery/efd6fb31-137b-41e3-b32d-438594e51a18)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:23:38.080]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:23:34.8927473Z);\\nlet _endTime = datetime(2026-08-24T22:23:34.8927477Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrgztftd846p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:23:38.0025191Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;025d76c9-ff35-453c-a1d4-d6e1aa8c5ae1\\\",\\r\\n \\\"activityId\\\": \\\"ef32b257-f29d-4b0c-81be-6fd6150546f1\\\",\\r\\n \\\"subActivityId\\\": \\\"efd6fb31-137b-41e3-b32d-438594e51a18\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bbf6b235-380f-4084-8005-856fcc5a3fc6\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;025d76c9-ff35-453c-a1d4-d6e1aa8c5ae1 ARID=ef32b257-f29d-4b0c-81be-6fd6150546f1 \\u003e GW.Http.CallContext/ef32b257-f29d-4b0c-81be-6fd6150546f1 \\u003e GW.ExecuteQuery/bbf6b235-380f-4084-8005-856fcc5a3fc6 \\u003e DN.FE.ExecuteQuery/efd6fb31-137b-41e3-b32d-438594e51a18)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[22:23:45.286]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:23:41.6923571Z);\\nlet _endTime = datetime(2026-08-24T22:23:41.6923576Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrgztftd846p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:23:45.2650146Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2e4b972d-ca6b-440c-8e53-44a9547bd21d\\\",\\r\\n \\\"activityId\\\": \\\"68da6022-33f6-4d2a-8f0b-b8f75a25481d\\\",\\r\\n \\\"subActivityId\\\": \\\"ab90ba5e-3045-41f0-a9b2-b9bec85664f5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e5180dcb-f6ed-4270-b95c-603ec7e8fcd0\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2e4b972d-ca6b-440c-8e53-44a9547bd21d ARID=68da6022-33f6-4d2a-8f0b-b8f75a25481d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e23736d1-fb0e-41f0-9f0e-c1d42a7dabbe \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e23736d1-fb0e-41f0-9f0e-c1d42a7dabbe \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5180dcb-f6ed-4270-b95c-603ec7e8fcd0 \\u003e DN.FE.ExecuteQuery/ab90ba5e-3045-41f0-a9b2-b9bec85664f5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:23:45.384]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:23:41.6923571Z);\\nlet _endTime = datetime(2026-08-24T22:23:41.6923576Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrgztftd846p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:23:45.2650146Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2e4b972d-ca6b-440c-8e53-44a9547bd21d\\\",\\r\\n \\\"activityId\\\": \\\"68da6022-33f6-4d2a-8f0b-b8f75a25481d\\\",\\r\\n \\\"subActivityId\\\": \\\"ab90ba5e-3045-41f0-a9b2-b9bec85664f5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e5180dcb-f6ed-4270-b95c-603ec7e8fcd0\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2e4b972d-ca6b-440c-8e53-44a9547bd21d ARID=68da6022-33f6-4d2a-8f0b-b8f75a25481d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e23736d1-fb0e-41f0-9f0e-c1d42a7dabbe \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e23736d1-fb0e-41f0-9f0e-c1d42a7dabbe \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5180dcb-f6ed-4270-b95c-603ec7e8fcd0 \\u003e DN.FE.ExecuteQuery/ab90ba5e-3045-41f0-a9b2-b9bec85664f5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 618.943s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000aba840\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[22:23:34.777]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:23:31.1340475Z);\\nlet _endTime = datetime(2026-08-24T22:23:31.1340479Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrgztftd846p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:23:34.7544299Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;29db54ce-825b-4bb8-9a70-02b555f68f68\\\",\\r\\n \\\"activityId\\\": \\\"64a8c8f1-b43e-49c5-b452-340f5dfc4e2c\\\",\\r\\n \\\"subActivityId\\\": \\\"705e5e56-0d99-4165-83da-d9db2f1f98e3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c1332563-9620-4ce8-b789-424edc8d3469\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;29db54ce-825b-4bb8-9a70-02b555f68f68 ARID=64a8c8f1-b43e-49c5-b452-340f5dfc4e2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/fe5bfcde-d8a3-480b-8a59-62f13d3daed7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/fe5bfcde-d8a3-480b-8a59-62f13d3daed7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/c1332563-9620-4ce8-b789-424edc8d3469 \\u003e DN.FE.ExecuteQuery/705e5e56-0d99-4165-83da-d9db2f1f98e3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:23:34.823]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:23:31.1340475Z);\\nlet _endTime = datetime(2026-08-24T22:23:31.1340479Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrgztftd846p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:23:34.7544299Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;29db54ce-825b-4bb8-9a70-02b555f68f68\\\",\\r\\n \\\"activityId\\\": \\\"64a8c8f1-b43e-49c5-b452-340f5dfc4e2c\\\",\\r\\n \\\"subActivityId\\\": \\\"705e5e56-0d99-4165-83da-d9db2f1f98e3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c1332563-9620-4ce8-b789-424edc8d3469\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;29db54ce-825b-4bb8-9a70-02b555f68f68 ARID=64a8c8f1-b43e-49c5-b452-340f5dfc4e2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/fe5bfcde-d8a3-480b-8a59-62f13d3daed7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/fe5bfcde-d8a3-480b-8a59-62f13d3daed7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/c1332563-9620-4ce8-b789-424edc8d3469 \\u003e DN.FE.ExecuteQuery/705e5e56-0d99-4165-83da-d9db2f1f98e3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[22:23:38.026]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:23:34.8927473Z);\\nlet _endTime = datetime(2026-08-24T22:23:34.8927477Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrgztftd846p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:23:38.0025191Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;025d76c9-ff35-453c-a1d4-d6e1aa8c5ae1\\\",\\r\\n \\\"activityId\\\": \\\"ef32b257-f29d-4b0c-81be-6fd6150546f1\\\",\\r\\n \\\"subActivityId\\\": \\\"efd6fb31-137b-41e3-b32d-438594e51a18\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bbf6b235-380f-4084-8005-856fcc5a3fc6\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;025d76c9-ff35-453c-a1d4-d6e1aa8c5ae1 ARID=ef32b257-f29d-4b0c-81be-6fd6150546f1 \\u003e GW.Http.CallContext/ef32b257-f29d-4b0c-81be-6fd6150546f1 \\u003e GW.ExecuteQuery/bbf6b235-380f-4084-8005-856fcc5a3fc6 \\u003e DN.FE.ExecuteQuery/efd6fb31-137b-41e3-b32d-438594e51a18)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:23:38.080]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:23:34.8927473Z);\\nlet _endTime = datetime(2026-08-24T22:23:34.8927477Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrgztftd846p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:23:38.0025191Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;025d76c9-ff35-453c-a1d4-d6e1aa8c5ae1\\\",\\r\\n \\\"activityId\\\": \\\"ef32b257-f29d-4b0c-81be-6fd6150546f1\\\",\\r\\n \\\"subActivityId\\\": \\\"efd6fb31-137b-41e3-b32d-438594e51a18\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bbf6b235-380f-4084-8005-856fcc5a3fc6\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;025d76c9-ff35-453c-a1d4-d6e1aa8c5ae1 ARID=ef32b257-f29d-4b0c-81be-6fd6150546f1 \\u003e GW.Http.CallContext/ef32b257-f29d-4b0c-81be-6fd6150546f1 \\u003e GW.ExecuteQuery/bbf6b235-380f-4084-8005-856fcc5a3fc6 \\u003e DN.FE.ExecuteQuery/efd6fb31-137b-41e3-b32d-438594e51a18)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[22:23:45.286]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:23:41.6923571Z);\\nlet _endTime = datetime(2026-08-24T22:23:41.6923576Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrgztftd846p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:23:45.2650146Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2e4b972d-ca6b-440c-8e53-44a9547bd21d\\\",\\r\\n \\\"activityId\\\": \\\"68da6022-33f6-4d2a-8f0b-b8f75a25481d\\\",\\r\\n \\\"subActivityId\\\": \\\"ab90ba5e-3045-41f0-a9b2-b9bec85664f5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e5180dcb-f6ed-4270-b95c-603ec7e8fcd0\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2e4b972d-ca6b-440c-8e53-44a9547bd21d ARID=68da6022-33f6-4d2a-8f0b-b8f75a25481d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e23736d1-fb0e-41f0-9f0e-c1d42a7dabbe \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e23736d1-fb0e-41f0-9f0e-c1d42a7dabbe \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5180dcb-f6ed-4270-b95c-603ec7e8fcd0 \\u003e DN.FE.ExecuteQuery/ab90ba5e-3045-41f0-a9b2-b9bec85664f5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:23:45.384]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:23:41.6923571Z);\\nlet _endTime = datetime(2026-08-24T22:23:41.6923576Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrgztftd846p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:23:45.2650146Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2e4b972d-ca6b-440c-8e53-44a9547bd21d\\\",\\r\\n \\\"activityId\\\": \\\"68da6022-33f6-4d2a-8f0b-b8f75a25481d\\\",\\r\\n \\\"subActivityId\\\": \\\"ab90ba5e-3045-41f0-a9b2-b9bec85664f5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e5180dcb-f6ed-4270-b95c-603ec7e8fcd0\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2e4b972d-ca6b-440c-8e53-44a9547bd21d ARID=68da6022-33f6-4d2a-8f0b-b8f75a25481d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e23736d1-fb0e-41f0-9f0e-c1d42a7dabbe \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e23736d1-fb0e-41f0-9f0e-c1d42a7dabbe \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5180dcb-f6ed-4270-b95c-603ec7e8fcd0 \\u003e DN.FE.ExecuteQuery/ab90ba5e-3045-41f0-a9b2-b9bec85664f5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091995273302642688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6648,"pr_state":"open","pr_sha":"d5a9f482aee0f501c8ae8bda8cd06ffa03214aaf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T21:05:15Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"50f7eb82ac98fb2aeea41f5d3a767a2580055fd87bf4c925d2fe9c483e8cdd40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091995273302642688","occurred_at":"2026-08-24T21:05:15Z","failed_at":"alert","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T22:22:12Z\n Ended: 2026-08-24T22:46:11Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j2642688-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"9be3204bf8f9ad7b8676f2ce3c742da5203ab6f2e6e522501838dc7238814b22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091995273302642688","occurred_at":"2026-08-24T21:05:15Z","failed_at":"e2e","signature_id":"b19e1030a263db434fc5938127062d3add16c8d3a56319f83b1663785d30bba9","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000e70850\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[22:29:34.250]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:29:31.0597247Z);\\nlet _endTime = datetime(2026-08-24T22:29:31.0597254Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-g8cd8qls4hh6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:29:34.2294855Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ada62b8a-d2fa-4201-bf57-a131a1316d95\\\",\\r\\n \\\"activityId\\\": \\\"384e9969-d946-4906-9998-c82ec6a77993\\\",\\r\\n \\\"subActivityId\\\": \\\"789ba336-45e5-4f69-84c4-f02612a2f6cc\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f0abbf07-bd6a-4728-acfe-5cef65fcda63\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ada62b8a-d2fa-4201-bf57-a131a1316d95 ARID=384e9969-d946-4906-9998-c82ec6a77993 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b13ab986-6ab1-4d9f-bb60-653a61d05f70 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b13ab986-6ab1-4d9f-bb60-653a61d05f70 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f0abbf07-bd6a-4728-acfe-5cef65fcda63 \\u003e DN.FE.ExecuteQuery/789ba336-45e5-4f69-84c4-f02612a2f6cc)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:29:34.301]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:29:31.0597247Z);\\nlet _endTime = datetime(2026-08-24T22:29:31.0597254Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-g8cd8qls4hh6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:29:34.2294855Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ada62b8a-d2fa-4201-bf57-a131a1316d95\\\",\\r\\n \\\"activityId\\\": \\\"384e9969-d946-4906-9998-c82ec6a77993\\\",\\r\\n \\\"subActivityId\\\": \\\"789ba336-45e5-4f69-84c4-f02612a2f6cc\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f0abbf07-bd6a-4728-acfe-5cef65fcda63\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ada62b8a-d2fa-4201-bf57-a131a1316d95 ARID=384e9969-d946-4906-9998-c82ec6a77993 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b13ab986-6ab1-4d9f-bb60-653a61d05f70 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b13ab986-6ab1-4d9f-bb60-653a61d05f70 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f0abbf07-bd6a-4728-acfe-5cef65fcda63 \\u003e DN.FE.ExecuteQuery/789ba336-45e5-4f69-84c4-f02612a2f6cc)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[22:29:37.454]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:29:34.3746843Z);\\nlet _endTime = datetime(2026-08-24T22:29:34.3746850Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-g8cd8qls4hh6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:29:37.4330362Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c0841516-1b82-4da0-8610-d8a2b44299a3\\\",\\r\\n \\\"activityId\\\": \\\"6ba9ca04-11a9-44fa-b3d6-0a774b4600bc\\\",\\r\\n \\\"subActivityId\\\": \\\"53e784a5-c3af-4317-954e-e58a974cee6d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"56500fba-f877-4e8b-802e-b37b3fd70d19\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c0841516-1b82-4da0-8610-d8a2b44299a3 ARID=6ba9ca04-11a9-44fa-b3d6-0a774b4600bc \\u003e GW.Http.CallContext/6ba9ca04-11a9-44fa-b3d6-0a774b4600bc \\u003e GW.ExecuteQuery/56500fba-f877-4e8b-802e-b37b3fd70d19 \\u003e DN.FE.ExecuteQuery/53e784a5-c3af-4317-954e-e58a974cee6d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:29:37.504]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:29:34.3746843Z);\\nlet _endTime = datetime(2026-08-24T22:29:34.3746850Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-g8cd8qls4hh6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:29:37.4330362Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c0841516-1b82-4da0-8610-d8a2b44299a3\\\",\\r\\n \\\"activityId\\\": \\\"6ba9ca04-11a9-44fa-b3d6-0a774b4600bc\\\",\\r\\n \\\"subActivityId\\\": \\\"53e784a5-c3af-4317-954e-e58a974cee6d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"56500fba-f877-4e8b-802e-b37b3fd70d19\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c0841516-1b82-4da0-8610-d8a2b44299a3 ARID=6ba9ca04-11a9-44fa-b3d6-0a774b4600bc \\u003e GW.Http.CallContext/6ba9ca04-11a9-44fa-b3d6-0a774b4600bc \\u003e GW.ExecuteQuery/56500fba-f877-4e8b-802e-b37b3fd70d19 \\u003e DN.FE.ExecuteQuery/53e784a5-c3af-4317-954e-e58a974cee6d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[22:29:44.411]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:29:40.9730143Z);\\nlet _endTime = datetime(2026-08-24T22:29:40.9730148Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-g8cd8qls4hh6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:29:44.3871339Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6c96089c-3f08-4ae2-84cd-e6920c98e2f4\\\",\\r\\n \\\"activityId\\\": \\\"cda408ef-22ce-4bd6-bad1-04a3011f4f86\\\",\\r\\n \\\"subActivityId\\\": \\\"49ad97cc-b49e-4c1d-a94b-bdee0e82195a\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"67c19916-b811-4022-8b94-037ea0c81eae\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6c96089c-3f08-4ae2-84cd-e6920c98e2f4 ARID=cda408ef-22ce-4bd6-bad1-04a3011f4f86 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b6df6c94-d688-4ac3-986f-aeaddd3706e7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b6df6c94-d688-4ac3-986f-aeaddd3706e7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/67c19916-b811-4022-8b94-037ea0c81eae \\u003e DN.FE.ExecuteQuery/49ad97cc-b49e-4c1d-a94b-bdee0e82195a)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:29:44.518]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:29:40.9730143Z);\\nlet _endTime = datetime(2026-08-24T22:29:40.9730148Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-g8cd8qls4hh6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:29:44.3871339Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6c96089c-3f08-4ae2-84cd-e6920c98e2f4\\\",\\r\\n \\\"activityId\\\": \\\"cda408ef-22ce-4bd6-bad1-04a3011f4f86\\\",\\r\\n \\\"subActivityId\\\": \\\"49ad97cc-b49e-4c1d-a94b-bdee0e82195a\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"67c19916-b811-4022-8b94-037ea0c81eae\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6c96089c-3f08-4ae2-84cd-e6920c98e2f4 ARID=cda408ef-22ce-4bd6-bad1-04a3011f4f86 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b6df6c94-d688-4ac3-986f-aeaddd3706e7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b6df6c94-d688-4ac3-986f-aeaddd3706e7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/67c19916-b811-4022-8b94-037ea0c81eae \\u003e DN.FE.ExecuteQuery/49ad97cc-b49e-4c1d-a94b-bdee0e82195a)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000e70850\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[22:29:34.250]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:29:31.0597247Z);\\nlet _endTime = datetime(2026-08-24T22:29:31.0597254Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-g8cd8qls4hh6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:29:34.2294855Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ada62b8a-d2fa-4201-bf57-a131a1316d95\\\",\\r\\n \\\"activityId\\\": \\\"384e9969-d946-4906-9998-c82ec6a77993\\\",\\r\\n \\\"subActivityId\\\": \\\"789ba336-45e5-4f69-84c4-f02612a2f6cc\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f0abbf07-bd6a-4728-acfe-5cef65fcda63\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ada62b8a-d2fa-4201-bf57-a131a1316d95 ARID=384e9969-d946-4906-9998-c82ec6a77993 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b13ab986-6ab1-4d9f-bb60-653a61d05f70 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b13ab986-6ab1-4d9f-bb60-653a61d05f70 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f0abbf07-bd6a-4728-acfe-5cef65fcda63 \\u003e DN.FE.ExecuteQuery/789ba336-45e5-4f69-84c4-f02612a2f6cc)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:29:34.301]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:29:31.0597247Z);\\nlet _endTime = datetime(2026-08-24T22:29:31.0597254Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-g8cd8qls4hh6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:29:34.2294855Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ada62b8a-d2fa-4201-bf57-a131a1316d95\\\",\\r\\n \\\"activityId\\\": \\\"384e9969-d946-4906-9998-c82ec6a77993\\\",\\r\\n \\\"subActivityId\\\": \\\"789ba336-45e5-4f69-84c4-f02612a2f6cc\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f0abbf07-bd6a-4728-acfe-5cef65fcda63\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ada62b8a-d2fa-4201-bf57-a131a1316d95 ARID=384e9969-d946-4906-9998-c82ec6a77993 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b13ab986-6ab1-4d9f-bb60-653a61d05f70 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b13ab986-6ab1-4d9f-bb60-653a61d05f70 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f0abbf07-bd6a-4728-acfe-5cef65fcda63 \\u003e DN.FE.ExecuteQuery/789ba336-45e5-4f69-84c4-f02612a2f6cc)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[22:29:37.454]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:29:34.3746843Z);\\nlet _endTime = datetime(2026-08-24T22:29:34.3746850Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-g8cd8qls4hh6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:29:37.4330362Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c0841516-1b82-4da0-8610-d8a2b44299a3\\\",\\r\\n \\\"activityId\\\": \\\"6ba9ca04-11a9-44fa-b3d6-0a774b4600bc\\\",\\r\\n \\\"subActivityId\\\": \\\"53e784a5-c3af-4317-954e-e58a974cee6d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"56500fba-f877-4e8b-802e-b37b3fd70d19\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c0841516-1b82-4da0-8610-d8a2b44299a3 ARID=6ba9ca04-11a9-44fa-b3d6-0a774b4600bc \\u003e GW.Http.CallContext/6ba9ca04-11a9-44fa-b3d6-0a774b4600bc \\u003e GW.ExecuteQuery/56500fba-f877-4e8b-802e-b37b3fd70d19 \\u003e DN.FE.ExecuteQuery/53e784a5-c3af-4317-954e-e58a974cee6d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:29:37.504]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:29:34.3746843Z);\\nlet _endTime = datetime(2026-08-24T22:29:34.3746850Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-g8cd8qls4hh6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:29:37.4330362Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c0841516-1b82-4da0-8610-d8a2b44299a3\\\",\\r\\n \\\"activityId\\\": \\\"6ba9ca04-11a9-44fa-b3d6-0a774b4600bc\\\",\\r\\n \\\"subActivityId\\\": \\\"53e784a5-c3af-4317-954e-e58a974cee6d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"56500fba-f877-4e8b-802e-b37b3fd70d19\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c0841516-1b82-4da0-8610-d8a2b44299a3 ARID=6ba9ca04-11a9-44fa-b3d6-0a774b4600bc \\u003e GW.Http.CallContext/6ba9ca04-11a9-44fa-b3d6-0a774b4600bc \\u003e GW.ExecuteQuery/56500fba-f877-4e8b-802e-b37b3fd70d19 \\u003e DN.FE.ExecuteQuery/53e784a5-c3af-4317-954e-e58a974cee6d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[22:29:44.411]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:29:40.9730143Z);\\nlet _endTime = datetime(2026-08-24T22:29:40.9730148Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-g8cd8qls4hh6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:29:44.3871339Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6c96089c-3f08-4ae2-84cd-e6920c98e2f4\\\",\\r\\n \\\"activityId\\\": \\\"cda408ef-22ce-4bd6-bad1-04a3011f4f86\\\",\\r\\n \\\"subActivityId\\\": \\\"49ad97cc-b49e-4c1d-a94b-bdee0e82195a\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"67c19916-b811-4022-8b94-037ea0c81eae\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6c96089c-3f08-4ae2-84cd-e6920c98e2f4 ARID=cda408ef-22ce-4bd6-bad1-04a3011f4f86 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b6df6c94-d688-4ac3-986f-aeaddd3706e7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b6df6c94-d688-4ac3-986f-aeaddd3706e7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/67c19916-b811-4022-8b94-037ea0c81eae \\u003e DN.FE.ExecuteQuery/49ad97cc-b49e-4c1d-a94b-bdee0e82195a)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:29:44.518]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:29:40.9730143Z);\\nlet _endTime = datetime(2026-08-24T22:29:40.9730148Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-g8cd8qls4hh6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:29:44.3871339Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6c96089c-3f08-4ae2-84cd-e6920c98e2f4\\\",\\r\\n \\\"activityId\\\": \\\"cda408ef-22ce-4bd6-bad1-04a3011f4f86\\\",\\r\\n \\\"subActivityId\\\": \\\"49ad97cc-b49e-4c1d-a94b-bdee0e82195a\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"67c19916-b811-4022-8b94-037ea0c81eae\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6c96089c-3f08-4ae2-84cd-e6920c98e2f4 ARID=cda408ef-22ce-4bd6-bad1-04a3011f4f86 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b6df6c94-d688-4ac3-986f-aeaddd3706e7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b6df6c94-d688-4ac3-986f-aeaddd3706e7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/67c19916-b811-4022-8b94-037ea0c81eae \\u003e DN.FE.ExecuteQuery/49ad97cc-b49e-4c1d-a94b-bdee0e82195a)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6660,"pr_state":"open","pr_sha":"a660b6e55fc0b5bbac8bc93588a1797519a9c76f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T21:03:33Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"0acefdc5704d87a9a35fc8c1c593d149588dc7f48c4f1e6ce7a4fe3848983566","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","failed_at":"e2e","signature_id":"7cc1252e9b8fe966bbe77836b76cbb84f92a735b546bfadc6681cd7399872b22","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138e500\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-fddhzl9zg9xz\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e47f8d6-28af-497c-8094-400544739634\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e47f8d6-28af-497c-8094-400544739634\",\n \"name\": \"9e47f8d6-28af-497c-8094-400544739634\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:51:08.657608962Z\",\n \"endTime\": \"2026-08-24T22:10:16.835899624Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138e500\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-fddhzl9zg9xz\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e47f8d6-28af-497c-8094-400544739634\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e47f8d6-28af-497c-8094-400544739634\",\n \"name\": \"9e47f8d6-28af-497c-8094-400544739634\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:51:08.657608962Z\",\n \"endTime\": \"2026-08-24T22:10:16.835899624Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"14e718e5f117b79b69b428310b729ca4089ca6a9daea00152f65897bcd143b68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","failed_at":"e2e","signature_id":"e5919a5aa6e234b2c6b16539e4375d2ff9f7724d31d00cb9d8f2c6f3bd3b38ad","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e33220\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-ndmwcdqgk685\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a097060e-673b-40de-a82a-aa623235d88d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a097060e-673b-40de-a82a-aa623235d88d\",\n \"name\": \"a097060e-673b-40de-a82a-aa623235d88d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:51:06.650394368Z\",\n \"endTime\": \"2026-08-24T22:10:06.638990375Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e33220\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-ndmwcdqgk685\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a097060e-673b-40de-a82a-aa623235d88d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a097060e-673b-40de-a82a-aa623235d88d\",\n \"name\": \"a097060e-673b-40de-a82a-aa623235d88d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:51:06.650394368Z\",\n \"endTime\": \"2026-08-24T22:10:06.638990375Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"297879af6c7a3b5f3c37fb94ac2c22850f1c9194ae41ea1ea4cd608fc41b124b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","failed_at":"e2e","signature_id":"6487190a1499ee6eb043aa40a245add60c110f31922a104a2b7ccae5c75948f8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-bvzdjg8t8hhx/cluster-candidate-5-0-vn2bh6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eee460\u003e: \n failed to create HCP cluster cluster-candidate-5-0-vn2bh6: failed waiting for cluster=\"cluster-candidate-5-0-vn2bh6\" in resourcegroup=\"rg-candidate-5-0-bvzdjg8t8hhx\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7131e198-16ae-4d71-b14c-8fd65e02139b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7131e198-16ae-4d71-b14c-8fd65e02139b\",\n \"name\": \"7131e198-16ae-4d71-b14c-8fd65e02139b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:51:13.862127909Z\",\n \"endTime\": \"2026-08-24T22:10:17.159098241Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-bvzdjg8t8hhx/cluster-candidate-5-0-vn2bh6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eee460\u003e: \n failed to create HCP cluster cluster-candidate-5-0-vn2bh6: failed waiting for cluster=\"cluster-candidate-5-0-vn2bh6\" in resourcegroup=\"rg-candidate-5-0-bvzdjg8t8hhx\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7131e198-16ae-4d71-b14c-8fd65e02139b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7131e198-16ae-4d71-b14c-8fd65e02139b\",\n \"name\": \"7131e198-16ae-4d71-b14c-8fd65e02139b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:51:13.862127909Z\",\n \"endTime\": \"2026-08-24T22:10:17.159098241Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5c5f56f1fd56f0a06cbcbbbbae32be5a3a0fe69619b44c57ea5151588808a124","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","failed_at":"e2e","signature_id":"87055b770927de3226a00a72e6a1e8f425c4181b7305ba9fd94502a36df5fec7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-22lvs576ptrb/cluster-candidate-4-20-qdcf69 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001190ba0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-qdcf69: failed waiting for cluster=\"cluster-candidate-4-20-qdcf69\" in resourcegroup=\"rg-candidate-4-20-22lvs576ptrb\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc1dd563-6c7e-4d20-8fce-99e12ec22770\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc1dd563-6c7e-4d20-8fce-99e12ec22770\",\n \"name\": \"dc1dd563-6c7e-4d20-8fce-99e12ec22770\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:51:04.562343833Z\",\n \"endTime\": \"2026-08-24T22:10:06.652345811Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-22lvs576ptrb/cluster-candidate-4-20-qdcf69 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001190ba0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-qdcf69: failed waiting for cluster=\"cluster-candidate-4-20-qdcf69\" in resourcegroup=\"rg-candidate-4-20-22lvs576ptrb\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc1dd563-6c7e-4d20-8fce-99e12ec22770\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc1dd563-6c7e-4d20-8fce-99e12ec22770\",\n \"name\": \"dc1dd563-6c7e-4d20-8fce-99e12ec22770\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:51:04.562343833Z\",\n \"endTime\": \"2026-08-24T22:10:06.652345811Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"80872edc76097e91f6d896fbf1c89fc4132a9b475ea0b0f57e14b2dc1f0257df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","failed_at":"e2e","signature_id":"98f2c85bfff864b394f0a6540743e0876e4af34e9f265ec3b983e45e8ef64a57","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 631.434s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00059a560\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[22:16:23.288]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:16:18.5189597Z);\\nlet _endTime = datetime(2026-08-24T22:16:18.5189602Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vvg442pqd5g9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:16:23.2616891Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;4edcbff7-3ca0-4aa5-818e-511636b3ef0c\\\",\\r\\n \\\"activityId\\\": \\\"678135c6-f4c0-42d7-b677-90ad3afd7cab\\\",\\r\\n \\\"subActivityId\\\": \\\"f58a13bb-93db-4a42-90ea-ec66ad81b256\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"fd9a7b52-cd1f-44c5-8ff4-f320c1419c1b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4edcbff7-3ca0-4aa5-818e-511636b3ef0c ARID=678135c6-f4c0-42d7-b677-90ad3afd7cab \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7c7bbaad-c9f1-43c4-b0ac-c7b8807c51dd \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7c7bbaad-c9f1-43c4-b0ac-c7b8807c51dd \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/fd9a7b52-cd1f-44c5-8ff4-f320c1419c1b \\u003e DN.FE.ExecuteQuery/f58a13bb-93db-4a42-90ea-ec66ad81b256)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:16:23.343]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:16:18.5189597Z);\\nlet _endTime = datetime(2026-08-24T22:16:18.5189602Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vvg442pqd5g9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:16:23.2616891Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;4edcbff7-3ca0-4aa5-818e-511636b3ef0c\\\",\\r\\n \\\"activityId\\\": \\\"678135c6-f4c0-42d7-b677-90ad3afd7cab\\\",\\r\\n \\\"subActivityId\\\": \\\"f58a13bb-93db-4a42-90ea-ec66ad81b256\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"fd9a7b52-cd1f-44c5-8ff4-f320c1419c1b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4edcbff7-3ca0-4aa5-818e-511636b3ef0c ARID=678135c6-f4c0-42d7-b677-90ad3afd7cab \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7c7bbaad-c9f1-43c4-b0ac-c7b8807c51dd \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7c7bbaad-c9f1-43c4-b0ac-c7b8807c51dd \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/fd9a7b52-cd1f-44c5-8ff4-f320c1419c1b \\u003e DN.FE.ExecuteQuery/f58a13bb-93db-4a42-90ea-ec66ad81b256)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[22:16:28.739]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:16:23.4130746Z);\\nlet _endTime = datetime(2026-08-24T22:16:23.4130750Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vvg442pqd5g9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:16:28.7111996Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f5e99620-b1af-486e-8321-bf4f7de99b00\\\",\\r\\n \\\"activityId\\\": \\\"a7e14c65-4632-4a3e-86a5-525812d02bc7\\\",\\r\\n \\\"subActivityId\\\": \\\"932e42f2-36aa-416c-b097-ffd35a2d0018\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bd188a3a-ae2f-48fd-abfc-42d2265367c4\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f5e99620-b1af-486e-8321-bf4f7de99b00 ARID=a7e14c65-4632-4a3e-86a5-525812d02bc7 \\u003e GW.Http.CallContext/a7e14c65-4632-4a3e-86a5-525812d02bc7 \\u003e GW.ExecuteQuery/bd188a3a-ae2f-48fd-abfc-42d2265367c4 \\u003e DN.FE.ExecuteQuery/932e42f2-36aa-416c-b097-ffd35a2d0018)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:16:29.456]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:16:23.4130746Z);\\nlet _endTime = datetime(2026-08-24T22:16:23.4130750Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vvg442pqd5g9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:16:28.7111996Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f5e99620-b1af-486e-8321-bf4f7de99b00\\\",\\r\\n \\\"activityId\\\": \\\"a7e14c65-4632-4a3e-86a5-525812d02bc7\\\",\\r\\n \\\"subActivityId\\\": \\\"932e42f2-36aa-416c-b097-ffd35a2d0018\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bd188a3a-ae2f-48fd-abfc-42d2265367c4\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f5e99620-b1af-486e-8321-bf4f7de99b00 ARID=a7e14c65-4632-4a3e-86a5-525812d02bc7 \\u003e GW.Http.CallContext/a7e14c65-4632-4a3e-86a5-525812d02bc7 \\u003e GW.ExecuteQuery/bd188a3a-ae2f-48fd-abfc-42d2265367c4 \\u003e DN.FE.ExecuteQuery/932e42f2-36aa-416c-b097-ffd35a2d0018)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[22:16:45.979]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:16:38.6416976Z);\\nlet _endTime = datetime(2026-08-24T22:16:38.6416981Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vvg442pqd5g9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:16:45.8160979Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;adf5beb9-c32d-4096-9592-6c86179bcb5b\\\",\\r\\n \\\"activityId\\\": \\\"9920c01e-ffb2-406f-af33-8e776a18186b\\\",\\r\\n \\\"subActivityId\\\": \\\"23f5646a-c800-4fde-a433-c951f4fb3bb4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ce428594-daf1-4119-8de8-dc3f4edd8065\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;adf5beb9-c32d-4096-9592-6c86179bcb5b ARID=9920c01e-ffb2-406f-af33-8e776a18186b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/39567b3b-31f3-4b67-ad88-7e62bc36be0c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/39567b3b-31f3-4b67-ad88-7e62bc36be0c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ce428594-daf1-4119-8de8-dc3f4edd8065 \\u003e DN.FE.ExecuteQuery/23f5646a-c800-4fde-a433-c951f4fb3bb4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:16:47.476]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:16:38.6416976Z);\\nlet _endTime = datetime(2026-08-24T22:16:38.6416981Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vvg442pqd5g9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:16:45.8160979Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;adf5beb9-c32d-4096-9592-6c86179bcb5b\\\",\\r\\n \\\"activityId\\\": \\\"9920c01e-ffb2-406f-af33-8e776a18186b\\\",\\r\\n \\\"subActivityId\\\": \\\"23f5646a-c800-4fde-a433-c951f4fb3bb4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ce428594-daf1-4119-8de8-dc3f4edd8065\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;adf5beb9-c32d-4096-9592-6c86179bcb5b ARID=9920c01e-ffb2-406f-af33-8e776a18186b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/39567b3b-31f3-4b67-ad88-7e62bc36be0c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/39567b3b-31f3-4b67-ad88-7e62bc36be0c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ce428594-daf1-4119-8de8-dc3f4edd8065 \\u003e DN.FE.ExecuteQuery/23f5646a-c800-4fde-a433-c951f4fb3bb4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 631.434s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00059a560\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[22:16:23.288]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:16:18.5189597Z);\\nlet _endTime = datetime(2026-08-24T22:16:18.5189602Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vvg442pqd5g9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:16:23.2616891Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;4edcbff7-3ca0-4aa5-818e-511636b3ef0c\\\",\\r\\n \\\"activityId\\\": \\\"678135c6-f4c0-42d7-b677-90ad3afd7cab\\\",\\r\\n \\\"subActivityId\\\": \\\"f58a13bb-93db-4a42-90ea-ec66ad81b256\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"fd9a7b52-cd1f-44c5-8ff4-f320c1419c1b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4edcbff7-3ca0-4aa5-818e-511636b3ef0c ARID=678135c6-f4c0-42d7-b677-90ad3afd7cab \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7c7bbaad-c9f1-43c4-b0ac-c7b8807c51dd \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7c7bbaad-c9f1-43c4-b0ac-c7b8807c51dd \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/fd9a7b52-cd1f-44c5-8ff4-f320c1419c1b \\u003e DN.FE.ExecuteQuery/f58a13bb-93db-4a42-90ea-ec66ad81b256)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:16:23.343]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:16:18.5189597Z);\\nlet _endTime = datetime(2026-08-24T22:16:18.5189602Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vvg442pqd5g9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:16:23.2616891Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;4edcbff7-3ca0-4aa5-818e-511636b3ef0c\\\",\\r\\n \\\"activityId\\\": \\\"678135c6-f4c0-42d7-b677-90ad3afd7cab\\\",\\r\\n \\\"subActivityId\\\": \\\"f58a13bb-93db-4a42-90ea-ec66ad81b256\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"fd9a7b52-cd1f-44c5-8ff4-f320c1419c1b\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4edcbff7-3ca0-4aa5-818e-511636b3ef0c ARID=678135c6-f4c0-42d7-b677-90ad3afd7cab \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7c7bbaad-c9f1-43c4-b0ac-c7b8807c51dd \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7c7bbaad-c9f1-43c4-b0ac-c7b8807c51dd \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/fd9a7b52-cd1f-44c5-8ff4-f320c1419c1b \\u003e DN.FE.ExecuteQuery/f58a13bb-93db-4a42-90ea-ec66ad81b256)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[22:16:28.739]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:16:23.4130746Z);\\nlet _endTime = datetime(2026-08-24T22:16:23.4130750Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vvg442pqd5g9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:16:28.7111996Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f5e99620-b1af-486e-8321-bf4f7de99b00\\\",\\r\\n \\\"activityId\\\": \\\"a7e14c65-4632-4a3e-86a5-525812d02bc7\\\",\\r\\n \\\"subActivityId\\\": \\\"932e42f2-36aa-416c-b097-ffd35a2d0018\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bd188a3a-ae2f-48fd-abfc-42d2265367c4\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f5e99620-b1af-486e-8321-bf4f7de99b00 ARID=a7e14c65-4632-4a3e-86a5-525812d02bc7 \\u003e GW.Http.CallContext/a7e14c65-4632-4a3e-86a5-525812d02bc7 \\u003e GW.ExecuteQuery/bd188a3a-ae2f-48fd-abfc-42d2265367c4 \\u003e DN.FE.ExecuteQuery/932e42f2-36aa-416c-b097-ffd35a2d0018)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:16:29.456]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:16:23.4130746Z);\\nlet _endTime = datetime(2026-08-24T22:16:23.4130750Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vvg442pqd5g9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:16:28.7111996Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f5e99620-b1af-486e-8321-bf4f7de99b00\\\",\\r\\n \\\"activityId\\\": \\\"a7e14c65-4632-4a3e-86a5-525812d02bc7\\\",\\r\\n \\\"subActivityId\\\": \\\"932e42f2-36aa-416c-b097-ffd35a2d0018\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"bd188a3a-ae2f-48fd-abfc-42d2265367c4\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f5e99620-b1af-486e-8321-bf4f7de99b00 ARID=a7e14c65-4632-4a3e-86a5-525812d02bc7 \\u003e GW.Http.CallContext/a7e14c65-4632-4a3e-86a5-525812d02bc7 \\u003e GW.ExecuteQuery/bd188a3a-ae2f-48fd-abfc-42d2265367c4 \\u003e DN.FE.ExecuteQuery/932e42f2-36aa-416c-b097-ffd35a2d0018)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[22:16:45.979]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:16:38.6416976Z);\\nlet _endTime = datetime(2026-08-24T22:16:38.6416981Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vvg442pqd5g9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:16:45.8160979Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;adf5beb9-c32d-4096-9592-6c86179bcb5b\\\",\\r\\n \\\"activityId\\\": \\\"9920c01e-ffb2-406f-af33-8e776a18186b\\\",\\r\\n \\\"subActivityId\\\": \\\"23f5646a-c800-4fde-a433-c951f4fb3bb4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ce428594-daf1-4119-8de8-dc3f4edd8065\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;adf5beb9-c32d-4096-9592-6c86179bcb5b ARID=9920c01e-ffb2-406f-af33-8e776a18186b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/39567b3b-31f3-4b67-ad88-7e62bc36be0c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/39567b3b-31f3-4b67-ad88-7e62bc36be0c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ce428594-daf1-4119-8de8-dc3f4edd8065 \\u003e DN.FE.ExecuteQuery/23f5646a-c800-4fde-a433-c951f4fb3bb4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[22:16:47.476]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T22:16:38.6416976Z);\\nlet _endTime = datetime(2026-08-24T22:16:38.6416981Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-vvg442pqd5g9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T22:16:45.8160979Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;adf5beb9-c32d-4096-9592-6c86179bcb5b\\\",\\r\\n \\\"activityId\\\": \\\"9920c01e-ffb2-406f-af33-8e776a18186b\\\",\\r\\n \\\"subActivityId\\\": \\\"23f5646a-c800-4fde-a433-c951f4fb3bb4\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ce428594-daf1-4119-8de8-dc3f4edd8065\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;adf5beb9-c32d-4096-9592-6c86179bcb5b ARID=9920c01e-ffb2-406f-af33-8e776a18186b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/39567b3b-31f3-4b67-ad88-7e62bc36be0c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/39567b3b-31f3-4b67-ad88-7e62bc36be0c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ce428594-daf1-4119-8de8-dc3f4edd8065 \\u003e DN.FE.ExecuteQuery/23f5646a-c800-4fde-a433-c951f4fb3bb4)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"9ad2d1e903ed6517519e2682ef0e8e35c0dde65778851993a17c15cb6e6f97f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","failed_at":"alert","signature_id":"762c894801e83a205ad61ad7c1a1171fcbbd7db65d82ac38613bccfd72e8ae0d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T22:04:58Z\n Ended: 2026-08-24T22:16:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9409536-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd6tku01d4e5c0s0k2ocfdtf91cpr8l-d3q6r6q7q0m9f7w\", pod=\"router-57846fcf5b-n5mlw\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd6tku01d4e5c0s0k2ocfdtf91cpr8l-d3q6r6q7q0m9f7w/router-57846fcf5b-n5mlw has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T22:04:58Z\n Ended: 2026-08-24T22:16:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9409536-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd6td57g5fetk5hnhg543eg0jfi9lrt-j5s2f3c0f5x7m8z\", pod=\"router-7586ffccfd-4mvlg\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd6td57g5fetk5hnhg543eg0jfi9lrt-j5s2f3c0f5x7m8z/router-7586ffccfd-4mvlg has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"bae330bdc9d27d37a9b21c9e1463973b569437854c5e0a0d81f6da4b3a4f33d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","failed_at":"e2e","signature_id":"49911eb59aa73a3d7ebf02bdb4e56de7a616897c72e3ac7abe420c0c7a47e40e","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001338b40\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6bbwstsgczsz\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99d622cc-3702-4280-b31f-0b350e37d154\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99d622cc-3702-4280-b31f-0b350e37d154\",\n \"name\": \"99d622cc-3702-4280-b31f-0b350e37d154\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:51:03.771394068Z\",\n \"endTime\": \"2026-08-24T22:10:06.611292378Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001338b40\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6bbwstsgczsz\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99d622cc-3702-4280-b31f-0b350e37d154\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99d622cc-3702-4280-b31f-0b350e37d154\",\n \"name\": \"99d622cc-3702-4280-b31f-0b350e37d154\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:51:03.771394068Z\",\n \"endTime\": \"2026-08-24T22:10:06.611292378Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"be575ff88e7b28d2d6824a262d58ec492def068c326eb0012dea930b20915bad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","failed_at":"e2e","signature_id":"a9397a1f0aaba5d8ad982fe383aca1267736bcbd0d40f55741f427e85ee8b6d8","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--w9hvk7h29vl7/cluster-hs-pre-4vwvb8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001b9780\u003e: \n failed to create HCP cluster cluster-hs-pre-4vwvb8: failed waiting for cluster=\"cluster-hs-pre-4vwvb8\" in resourcegroup=\"rg-hs-pre--w9hvk7h29vl7\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ab5d6fca-4327-4d5b-8822-0b904b70d00b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ab5d6fca-4327-4d5b-8822-0b904b70d00b\",\n \"name\": \"ab5d6fca-4327-4d5b-8822-0b904b70d00b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:50:43.487651081Z\",\n \"endTime\": \"2026-08-24T22:09:46.584847851Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--w9hvk7h29vl7/cluster-hs-pre-4vwvb8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001b9780\u003e: \n failed to create HCP cluster cluster-hs-pre-4vwvb8: failed waiting for cluster=\"cluster-hs-pre-4vwvb8\" in resourcegroup=\"rg-hs-pre--w9hvk7h29vl7\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ab5d6fca-4327-4d5b-8822-0b904b70d00b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ab5d6fca-4327-4d5b-8822-0b904b70d00b\",\n \"name\": \"ab5d6fca-4327-4d5b-8822-0b904b70d00b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:50:43.487651081Z\",\n \"endTime\": \"2026-08-24T22:09:46.584847851Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"3ec9ecd4e53729b55fe8f972a0acc75827b54561","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T20:43:22Z"},"failed_at":["alert","e2e"],"failed_test_count":9,"occurrences":[{"row_id":"1d2a24c35b89cd9fa7e3874dd4f65e3e5a5a249edaaf21b89c62dc3f8bc81b54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","failed_at":"e2e","signature_id":"4bba4cf482e3fa39dfa2ce3e799d7eb2c477fcc7e6ad0b0350124af0d131c053","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:148]: failed to create HCP cluster np-version-upgrade-cluster-vkwj8z with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bca9c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-vkwj8z: failed waiting for cluster=\"np-version-upgrade-cluster-vkwj8z\" in resourcegroup=\"rg-np-version-upgrade-vkwj8z-pvn2kjtfhwm4\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ca593c6-172c-442e-bdb4-2b96ae721c04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ca593c6-172c-442e-bdb4-2b96ae721c04\",\n \"name\": \"9ca593c6-172c-442e-bdb4-2b96ae721c04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:29:02.41159338Z\",\n \"endTime\": \"2026-08-24T21:48:06.346460728Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:148]: failed to create HCP cluster np-version-upgrade-cluster-vkwj8z with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bca9c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-vkwj8z: failed waiting for cluster=\"np-version-upgrade-cluster-vkwj8z\" in resourcegroup=\"rg-np-version-upgrade-vkwj8z-pvn2kjtfhwm4\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ca593c6-172c-442e-bdb4-2b96ae721c04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ca593c6-172c-442e-bdb4-2b96ae721c04\",\n \"name\": \"9ca593c6-172c-442e-bdb4-2b96ae721c04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:29:02.41159338Z\",\n \"endTime\": \"2026-08-24T21:48:06.346460728Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6da69f53827e243723972e01f99b181f79f179bb012b4c7f8ac0a2642f76d2c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","failed_at":"e2e","signature_id":"c5d8b0e5c56e2c31565dc9e53627bd3252420239c844111c2d9db849f9e9c51c","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:93]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a80440\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-kb45vnq4q6dl\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f3d7388-91da-4d85-961d-9ff931b5b33b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f3d7388-91da-4d85-961d-9ff931b5b33b\",\n \"name\": \"9f3d7388-91da-4d85-961d-9ff931b5b33b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:28:50.487332824Z\",\n \"endTime\": \"2026-08-24T21:47:56.254666995Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:93]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a80440\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-kb45vnq4q6dl\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f3d7388-91da-4d85-961d-9ff931b5b33b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f3d7388-91da-4d85-961d-9ff931b5b33b\",\n \"name\": \"9f3d7388-91da-4d85-961d-9ff931b5b33b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:28:50.487332824Z\",\n \"endTime\": \"2026-08-24T21:47:56.254666995Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6ed36cdf904338ae5b5a07b211d1de97468152fdb48a5c8c7f820f2f2f249acd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","failed_at":"e2e","signature_id":"fa1482e0c10fe89a03896ac5f14bc5ff11cbc2b5c024fd993535afa6403d322b","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 618.080s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0022162a0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:53:58.338]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:53:53.8591464Z);\\nlet _endTime = datetime(2026-08-24T21:53:53.8591470Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4448xk6286t9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:53:58.3085967Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ea05e4ed-8aab-4cbf-ac1f-16ee87838ed4\\\",\\r\\n \\\"activityId\\\": \\\"31110b47-1689-4099-b0ab-73175d25e75e\\\",\\r\\n \\\"subActivityId\\\": \\\"74c66fb0-9a3d-480c-9f12-67f385ace1d6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0c8fa664-22c8-4399-81ea-fdd064d89c4c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ea05e4ed-8aab-4cbf-ac1f-16ee87838ed4 ARID=31110b47-1689-4099-b0ab-73175d25e75e \\u003e GW.Http.CallContext/31110b47-1689-4099-b0ab-73175d25e75e \\u003e GW.ExecuteQuery/0c8fa664-22c8-4399-81ea-fdd064d89c4c \\u003e DN.FE.ExecuteQuery/74c66fb0-9a3d-480c-9f12-67f385ace1d6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:53:58.338]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"context canceled\",\n \"name\": \"cosmosResourceSnapshots\"\n }�[0m\n �[37m[21:53:58.430]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:53:53.8591464Z);\\nlet _endTime = datetime(2026-08-24T21:53:53.8591470Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4448xk6286t9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:53:58.3085967Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ea05e4ed-8aab-4cbf-ac1f-16ee87838ed4\\\",\\r\\n \\\"activityId\\\": \\\"31110b47-1689-4099-b0ab-73175d25e75e\\\",\\r\\n \\\"subActivityId\\\": \\\"74c66fb0-9a3d-480c-9f12-67f385ace1d6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0c8fa664-22c8-4399-81ea-fdd064d89c4c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ea05e4ed-8aab-4cbf-ac1f-16ee87838ed4 ARID=31110b47-1689-4099-b0ab-73175d25e75e \\u003e GW.Http.CallContext/31110b47-1689-4099-b0ab-73175d25e75e \\u003e GW.ExecuteQuery/0c8fa664-22c8-4399-81ea-fdd064d89c4c \\u003e DN.FE.ExecuteQuery/74c66fb0-9a3d-480c-9f12-67f385ace1d6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:54:02.554]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:53:58.6053160Z);\\nlet _endTime = datetime(2026-08-24T21:53:58.6053166Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4448xk6286t9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:54:02.5313557Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;57a9070e-2d96-422b-978f-0aacac128a23\\\",\\r\\n \\\"activityId\\\": \\\"6272b7b7-e9aa-464e-8a01-e9e1b3804dec\\\",\\r\\n \\\"subActivityId\\\": \\\"2bc2139c-c96a-4588-934c-86452b080a0b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"aac4e537-146b-4268-a5da-1277328712aa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;57a9070e-2d96-422b-978f-0aacac128a23 ARID=6272b7b7-e9aa-464e-8a01-e9e1b3804dec \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f8af6a8-db30-4af7-9126-fbbcbeb149c3 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f8af6a8-db30-4af7-9126-fbbcbeb149c3 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/aac4e537-146b-4268-a5da-1277328712aa \\u003e DN.FE.ExecuteQuery/2bc2139c-c96a-4588-934c-86452b080a0b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:54:02.554]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"context canceled\",\n \"name\": \"cosmosResourceSnapshots\"\n }�[0m\n �[37m[21:54:02.628]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:53:58.6053160Z);\\nlet _endTime = datetime(2026-08-24T21:53:58.6053166Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4448xk6286t9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:54:02.5313557Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;57a9070e-2d96-422b-978f-0aacac128a23\\\",\\r\\n \\\"activityId\\\": \\\"6272b7b7-e9aa-464e-8a01-e9e1b3804dec\\\",\\r\\n \\\"subActivityId\\\": \\\"2bc2139c-c96a-4588-934c-86452b080a0b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"aac4e537-146b-4268-a5da-1277328712aa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;57a9070e-2d96-422b-978f-0aacac128a23 ARID=6272b7b7-e9aa-464e-8a01-e9e1b3804dec \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f8af6a8-db30-4af7-9126-fbbcbeb149c3 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f8af6a8-db30-4af7-9126-fbbcbeb149c3 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/aac4e537-146b-4268-a5da-1277328712aa \\u003e DN.FE.ExecuteQuery/2bc2139c-c96a-4588-934c-86452b080a0b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:54:11.406]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:54:07.4434533Z);\\nlet _endTime = datetime(2026-08-24T21:54:07.4434539Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4448xk6286t9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:54:11.3785034Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;98a932a5-07da-4891-ae17-2c0a43b81d7d\\\",\\r\\n \\\"activityId\\\": \\\"de64d66e-7dd9-4de8-b52a-d9924cfcf263\\\",\\r\\n \\\"subActivityId\\\": \\\"1fe28bac-d3b5-487e-8112-1edd1a6e2076\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7bc313c6-07c1-4074-9710-48a250fa6b83\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;98a932a5-07da-4891-ae17-2c0a43b81d7d ARID=de64d66e-7dd9-4de8-b52a-d9924cfcf263 \\u003e GW.Http.CallContext/de64d66e-7dd9-4de8-b52a-d9924cfcf263 \\u003e GW.ExecuteQuery/7bc313c6-07c1-4074-9710-48a250fa6b83 \\u003e DN.FE.ExecuteQuery/1fe28bac-d3b5-487e-8112-1edd1a6e2076)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:54:11.407]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"context canceled\",\n \"name\": \"cosmosResourceSnapshots\"\n }�[0m\n �[37m[21:54:11.535]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:54:07.4434533Z);\\nlet _endTime = datetime(2026-08-24T21:54:07.4434539Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4448xk6286t9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:54:11.3785034Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;98a932a5-07da-4891-ae17-2c0a43b81d7d\\\",\\r\\n \\\"activityId\\\": \\\"de64d66e-7dd9-4de8-b52a-d9924cfcf263\\\",\\r\\n \\\"subActivityId\\\": \\\"1fe28bac-d3b5-487e-8112-1edd1a6e2076\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7bc313c6-07c1-4074-9710-48a250fa6b83\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;98a932a5-07da-4891-ae17-2c0a43b81d7d ARID=de64d66e-7dd9-4de8-b52a-d9924cfcf263 \\u003e GW.Http.CallContext/de64d66e-7dd9-4de8-b52a-d9924cfcf263 \\u003e GW.ExecuteQuery/7bc313c6-07c1-4074-9710-48a250fa6b83 \\u003e DN.FE.ExecuteQuery/1fe28bac-d3b5-487e-8112-1edd1a6e2076)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 618.080s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0022162a0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:53:58.338]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:53:53.8591464Z);\\nlet _endTime = datetime(2026-08-24T21:53:53.8591470Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4448xk6286t9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:53:58.3085967Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ea05e4ed-8aab-4cbf-ac1f-16ee87838ed4\\\",\\r\\n \\\"activityId\\\": \\\"31110b47-1689-4099-b0ab-73175d25e75e\\\",\\r\\n \\\"subActivityId\\\": \\\"74c66fb0-9a3d-480c-9f12-67f385ace1d6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0c8fa664-22c8-4399-81ea-fdd064d89c4c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ea05e4ed-8aab-4cbf-ac1f-16ee87838ed4 ARID=31110b47-1689-4099-b0ab-73175d25e75e \\u003e GW.Http.CallContext/31110b47-1689-4099-b0ab-73175d25e75e \\u003e GW.ExecuteQuery/0c8fa664-22c8-4399-81ea-fdd064d89c4c \\u003e DN.FE.ExecuteQuery/74c66fb0-9a3d-480c-9f12-67f385ace1d6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:53:58.338]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"context canceled\",\n \"name\": \"cosmosResourceSnapshots\"\n }�[0m\n �[37m[21:53:58.430]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:53:53.8591464Z);\\nlet _endTime = datetime(2026-08-24T21:53:53.8591470Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4448xk6286t9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:53:58.3085967Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ea05e4ed-8aab-4cbf-ac1f-16ee87838ed4\\\",\\r\\n \\\"activityId\\\": \\\"31110b47-1689-4099-b0ab-73175d25e75e\\\",\\r\\n \\\"subActivityId\\\": \\\"74c66fb0-9a3d-480c-9f12-67f385ace1d6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0c8fa664-22c8-4399-81ea-fdd064d89c4c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ea05e4ed-8aab-4cbf-ac1f-16ee87838ed4 ARID=31110b47-1689-4099-b0ab-73175d25e75e \\u003e GW.Http.CallContext/31110b47-1689-4099-b0ab-73175d25e75e \\u003e GW.ExecuteQuery/0c8fa664-22c8-4399-81ea-fdd064d89c4c \\u003e DN.FE.ExecuteQuery/74c66fb0-9a3d-480c-9f12-67f385ace1d6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:54:02.554]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:53:58.6053160Z);\\nlet _endTime = datetime(2026-08-24T21:53:58.6053166Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4448xk6286t9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:54:02.5313557Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;57a9070e-2d96-422b-978f-0aacac128a23\\\",\\r\\n \\\"activityId\\\": \\\"6272b7b7-e9aa-464e-8a01-e9e1b3804dec\\\",\\r\\n \\\"subActivityId\\\": \\\"2bc2139c-c96a-4588-934c-86452b080a0b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"aac4e537-146b-4268-a5da-1277328712aa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;57a9070e-2d96-422b-978f-0aacac128a23 ARID=6272b7b7-e9aa-464e-8a01-e9e1b3804dec \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f8af6a8-db30-4af7-9126-fbbcbeb149c3 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f8af6a8-db30-4af7-9126-fbbcbeb149c3 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/aac4e537-146b-4268-a5da-1277328712aa \\u003e DN.FE.ExecuteQuery/2bc2139c-c96a-4588-934c-86452b080a0b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:54:02.554]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"context canceled\",\n \"name\": \"cosmosResourceSnapshots\"\n }�[0m\n �[37m[21:54:02.628]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:53:58.6053160Z);\\nlet _endTime = datetime(2026-08-24T21:53:58.6053166Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4448xk6286t9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:54:02.5313557Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;57a9070e-2d96-422b-978f-0aacac128a23\\\",\\r\\n \\\"activityId\\\": \\\"6272b7b7-e9aa-464e-8a01-e9e1b3804dec\\\",\\r\\n \\\"subActivityId\\\": \\\"2bc2139c-c96a-4588-934c-86452b080a0b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"aac4e537-146b-4268-a5da-1277328712aa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;57a9070e-2d96-422b-978f-0aacac128a23 ARID=6272b7b7-e9aa-464e-8a01-e9e1b3804dec \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f8af6a8-db30-4af7-9126-fbbcbeb149c3 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f8af6a8-db30-4af7-9126-fbbcbeb149c3 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/aac4e537-146b-4268-a5da-1277328712aa \\u003e DN.FE.ExecuteQuery/2bc2139c-c96a-4588-934c-86452b080a0b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:54:11.406]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:54:07.4434533Z);\\nlet _endTime = datetime(2026-08-24T21:54:07.4434539Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4448xk6286t9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:54:11.3785034Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;98a932a5-07da-4891-ae17-2c0a43b81d7d\\\",\\r\\n \\\"activityId\\\": \\\"de64d66e-7dd9-4de8-b52a-d9924cfcf263\\\",\\r\\n \\\"subActivityId\\\": \\\"1fe28bac-d3b5-487e-8112-1edd1a6e2076\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7bc313c6-07c1-4074-9710-48a250fa6b83\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;98a932a5-07da-4891-ae17-2c0a43b81d7d ARID=de64d66e-7dd9-4de8-b52a-d9924cfcf263 \\u003e GW.Http.CallContext/de64d66e-7dd9-4de8-b52a-d9924cfcf263 \\u003e GW.ExecuteQuery/7bc313c6-07c1-4074-9710-48a250fa6b83 \\u003e DN.FE.ExecuteQuery/1fe28bac-d3b5-487e-8112-1edd1a6e2076)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:54:11.407]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"context canceled\",\n \"name\": \"cosmosResourceSnapshots\"\n }�[0m\n �[37m[21:54:11.535]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:54:07.4434533Z);\\nlet _endTime = datetime(2026-08-24T21:54:07.4434539Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4448xk6286t9'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:54:11.3785034Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;98a932a5-07da-4891-ae17-2c0a43b81d7d\\\",\\r\\n \\\"activityId\\\": \\\"de64d66e-7dd9-4de8-b52a-d9924cfcf263\\\",\\r\\n \\\"subActivityId\\\": \\\"1fe28bac-d3b5-487e-8112-1edd1a6e2076\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7bc313c6-07c1-4074-9710-48a250fa6b83\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;98a932a5-07da-4891-ae17-2c0a43b81d7d ARID=de64d66e-7dd9-4de8-b52a-d9924cfcf263 \\u003e GW.Http.CallContext/de64d66e-7dd9-4de8-b52a-d9924cfcf263 \\u003e GW.ExecuteQuery/7bc313c6-07c1-4074-9710-48a250fa6b83 \\u003e DN.FE.ExecuteQuery/1fe28bac-d3b5-487e-8112-1edd1a6e2076)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"83fc4990c170aa7a71160fd5965afbaaaae4563711a8d582b23de22ad2b2b464","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","failed_at":"e2e","signature_id":"409a88ec07ceec5d40d5ae23f780d5c74cad626824811548eedaa155d1eba475","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:109]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da71c0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-wqnnn79bk4qc\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e635655f-c9ac-44e1-8232-3550a4c20673\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e635655f-c9ac-44e1-8232-3550a4c20673\",\n \"name\": \"e635655f-c9ac-44e1-8232-3550a4c20673\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:28:57.457434277Z\",\n \"endTime\": \"2026-08-24T21:48:06.423004565Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:109]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da71c0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-wqnnn79bk4qc\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e635655f-c9ac-44e1-8232-3550a4c20673\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e635655f-c9ac-44e1-8232-3550a4c20673\",\n \"name\": \"e635655f-c9ac-44e1-8232-3550a4c20673\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:28:57.457434277Z\",\n \"endTime\": \"2026-08-24T21:48:06.423004565Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"991bb23d86952d0674265d5748e8a1704be12ae86edb3563425e3b74ce8893e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","failed_at":"e2e","signature_id":"7467fcd7d827463de8cbe0d7b5f551f437c25ba6ae7fc7de2730705826346049","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:304]: failed to create HCP cluster np-noedge-7pzvlw\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e28760\u003e: \n failed to create HCP cluster np-noedge-7pzvlw: failed waiting for cluster=\"np-noedge-7pzvlw\" in resourcegroup=\"rg-np-noedge-7pzvlw-hmhh49x6v5k9\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n \"name\": \"d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:29:01.361816726Z\",\n \"endTime\": \"2026-08-24T21:48:06.132587418Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:304]: failed to create HCP cluster np-noedge-7pzvlw\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e28760\u003e: \n failed to create HCP cluster np-noedge-7pzvlw: failed waiting for cluster=\"np-noedge-7pzvlw\" in resourcegroup=\"rg-np-noedge-7pzvlw-hmhh49x6v5k9\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n \"name\": \"d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:29:01.361816726Z\",\n \"endTime\": \"2026-08-24T21:48:06.132587418Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"34e193d1e971458756c5fedbfac4e10a25751030aa42d3492b58c65f0ceda0fd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bed8cb595eaa1679e105a87ab6059e5efcd4ec4b250c3669b13f9bf9ece6cc1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","failed_at":"alert","signature_id":"5a62ef9a7062e35c1e10c6c46839368f20045e53d18429e0c93ad1b8118f4ed4","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T21:39:20Z\n Ended: 2026-08-24T21:56:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6887040-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd6j294ump2ii7ijvchj6q14avo9t6j-s5a2x8g9m2u0n3v\", pod=\"router-b598fb5ff-bdw2b\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd6j294ump2ii7ijvchj6q14avo9t6j-s5a2x8g9m2u0n3v/router-b598fb5ff-bdw2b has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T21:40:21Z\n Ended: 2026-08-24T21:59:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6887040-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd6j9r400h8i6khifr1m2rhln38oon4-x8h6q0k2k5e2h7u\", pod=\"router-688776fb9c-d8gmt\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd6j9r400h8i6khifr1m2rhln38oon4-x8h6q0k2k5e2h7u/router-688776fb9c-d8gmt has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-24T21:42:19Z\n Ended: 2026-08-24T22:02:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6887040-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd6j7pdvj3o5q8qa4vm85iu5rt4rphu-k0n5x2x4a4h6q2k\", pod=\"router-69c5ffd8ff-bkcml\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd6j7pdvj3o5q8qa4vm85iu5rt4rphu-k0n5x2x4a4h6q2k/router-69c5ffd8ff-bkcml has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-24T21:43:18Z\n Ended: 2026-08-24T22:02:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6887040-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd6j4q996t8qenevmgfujnj2gprckvj-sre-hcp-cluster\", pod=\"router-768d6d4584-ph9kl\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd6j4q996t8qenevmgfujnj2gprckvj-sre-hcp-cluster/router-768d6d4584-ph9kl has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"bf07b7a70ec8a72b92ca076c0cb937f831a6d36d18b1eb6ed3ebdbc6130214d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","failed_at":"e2e","signature_id":"ca98190d0c65ef7e152d8e004d3f56a8bcff3a1f40939b19e5d08cfc9a992c4b","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:95]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e95be0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-7w4qb7bx58wj\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c4e3b7e-7834-4bf7-9d07-9139f539790f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c4e3b7e-7834-4bf7-9d07-9139f539790f\",\n \"name\": \"7c4e3b7e-7834-4bf7-9d07-9139f539790f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:28:50.851778461Z\",\n \"endTime\": \"2026-08-24T21:47:56.22619205Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:95]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e95be0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-7w4qb7bx58wj\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c4e3b7e-7834-4bf7-9d07-9139f539790f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c4e3b7e-7834-4bf7-9d07-9139f539790f\",\n \"name\": \"7c4e3b7e-7834-4bf7-9d07-9139f539790f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:28:50.851778461Z\",\n \"endTime\": \"2026-08-24T21:47:56.22619205Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"34e193d1e971458756c5fedbfac4e10a25751030aa42d3492b58c65f0ceda0fd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cc3d2edb15c814a176b217f52cc7160b8815414efe682981c2067de737eed4e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","failed_at":"e2e","signature_id":"5a1d95e1ce2afb3cd1318374aea17f4a6e4f5d30ef03f9f5a0cc441e9043a771","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-zdwzppg9xnjr/cluster-candidate-4-21-qkclqb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004e4e40\u003e: \n failed to create HCP cluster cluster-candidate-4-21-qkclqb: failed waiting for cluster=\"cluster-candidate-4-21-qkclqb\" in resourcegroup=\"rg-candidate-4-21-zdwzppg9xnjr\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2c316ff4-cfc3-45ef-9cbe-7261535c25c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2c316ff4-cfc3-45ef-9cbe-7261535c25c4\",\n \"name\": \"2c316ff4-cfc3-45ef-9cbe-7261535c25c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:29:10.414327144Z\",\n \"endTime\": \"2026-08-24T21:48:16.34399536Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-zdwzppg9xnjr/cluster-candidate-4-21-qkclqb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004e4e40\u003e: \n failed to create HCP cluster cluster-candidate-4-21-qkclqb: failed waiting for cluster=\"cluster-candidate-4-21-qkclqb\" in resourcegroup=\"rg-candidate-4-21-zdwzppg9xnjr\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2c316ff4-cfc3-45ef-9cbe-7261535c25c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2c316ff4-cfc3-45ef-9cbe-7261535c25c4\",\n \"name\": \"2c316ff4-cfc3-45ef-9cbe-7261535c25c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:29:10.414327144Z\",\n \"endTime\": \"2026-08-24T21:48:16.34399536Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e1f5b706fddfb3acdfd217aafdacdf7de64c42667d39d6491f9f944fde69fb49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","failed_at":"e2e","signature_id":"deae4888a32973190e55fe6f1f2ac8a8941dd6efae21aa17eaa50cc4448f5b70","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:148]: failed to create HCP cluster np-version-upgrade-cluster-cmkrj8 with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001254160\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-cmkrj8: failed waiting for cluster=\"np-version-upgrade-cluster-cmkrj8\" in resourcegroup=\"rg-np-version-upgrade-cmkrj8-rmmz8l24rr9j\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b6cb9847-c419-4682-a0ae-0563ed4844b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b6cb9847-c419-4682-a0ae-0563ed4844b9\",\n \"name\": \"b6cb9847-c419-4682-a0ae-0563ed4844b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:28:41.610970429Z\",\n \"endTime\": \"2026-08-24T21:47:46.226678817Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:148]: failed to create HCP cluster np-version-upgrade-cluster-cmkrj8 with version 4.21.30\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001254160\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-cmkrj8: failed waiting for cluster=\"np-version-upgrade-cluster-cmkrj8\" in resourcegroup=\"rg-np-version-upgrade-cmkrj8-rmmz8l24rr9j\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b6cb9847-c419-4682-a0ae-0563ed4844b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b6cb9847-c419-4682-a0ae-0563ed4844b9\",\n \"name\": \"b6cb9847-c419-4682-a0ae-0563ed4844b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:28:41.610970429Z\",\n \"endTime\": \"2026-08-24T21:47:46.226678817Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","34e193d1e971458756c5fedbfac4e10a25751030aa42d3492b58c65f0ceda0fd","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985911028912128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T20:28:03Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0b423bcdc4ac3c0f0cf108b6b17381f4cf0b932e657695f961ebba6e5751dca8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985911028912128","occurred_at":"2026-08-24T20:28:03Z","failed_at":"e2e","signature_id":"43910364f48f66523f1f9d08a417492faf101b0ab2cb444442ab1a553439c6b5","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 604.553s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001b12490\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:41:43.018]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:41:39.0159138Z);\\nlet _endTime = datetime(2026-08-24T21:41:39.0159143Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sdm9vzp4vv2s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:41:42.9938618Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;42b7fe0e-1ca7-4eb3-bd94-70b13c5f50a0\\\",\\r\\n \\\"activityId\\\": \\\"428709cb-5faa-460e-9fd0-19b365a38358\\\",\\r\\n \\\"subActivityId\\\": \\\"c6511ec0-fe87-45c4-8535-22ba47859a18\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"73e933ce-67eb-4635-9995-9c7ad9fa82a5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;42b7fe0e-1ca7-4eb3-bd94-70b13c5f50a0 ARID=428709cb-5faa-460e-9fd0-19b365a38358 \\u003e GW.Http.CallContext/428709cb-5faa-460e-9fd0-19b365a38358 \\u003e GW.ExecuteQuery/73e933ce-67eb-4635-9995-9c7ad9fa82a5 \\u003e DN.FE.ExecuteQuery/c6511ec0-fe87-45c4-8535-22ba47859a18)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:41:43.076]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:41:39.0159138Z);\\nlet _endTime = datetime(2026-08-24T21:41:39.0159143Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sdm9vzp4vv2s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:41:42.9938618Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;42b7fe0e-1ca7-4eb3-bd94-70b13c5f50a0\\\",\\r\\n \\\"activityId\\\": \\\"428709cb-5faa-460e-9fd0-19b365a38358\\\",\\r\\n \\\"subActivityId\\\": \\\"c6511ec0-fe87-45c4-8535-22ba47859a18\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"73e933ce-67eb-4635-9995-9c7ad9fa82a5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;42b7fe0e-1ca7-4eb3-bd94-70b13c5f50a0 ARID=428709cb-5faa-460e-9fd0-19b365a38358 \\u003e GW.Http.CallContext/428709cb-5faa-460e-9fd0-19b365a38358 \\u003e GW.ExecuteQuery/73e933ce-67eb-4635-9995-9c7ad9fa82a5 \\u003e DN.FE.ExecuteQuery/c6511ec0-fe87-45c4-8535-22ba47859a18)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:41:46.333]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:41:43.1465223Z);\\nlet _endTime = datetime(2026-08-24T21:41:43.1465227Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sdm9vzp4vv2s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:41:46.3064953Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;16d19bd3-8d48-4dca-aa85-db4c9607c99f\\\",\\r\\n \\\"activityId\\\": \\\"c7172e67-5da6-46b0-ac83-c1a5b5be4ded\\\",\\r\\n \\\"subActivityId\\\": \\\"a1170dba-d226-4772-89b6-9f45fb03d368\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d3feb915-cfb4-4185-8525-9185496fce8d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;16d19bd3-8d48-4dca-aa85-db4c9607c99f ARID=c7172e67-5da6-46b0-ac83-c1a5b5be4ded \\u003e GW.Http.CallContext/c7172e67-5da6-46b0-ac83-c1a5b5be4ded \\u003e GW.ExecuteQuery/d3feb915-cfb4-4185-8525-9185496fce8d \\u003e DN.FE.ExecuteQuery/a1170dba-d226-4772-89b6-9f45fb03d368)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:41:46.391]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:41:43.1465223Z);\\nlet _endTime = datetime(2026-08-24T21:41:43.1465227Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sdm9vzp4vv2s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:41:46.3064953Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;16d19bd3-8d48-4dca-aa85-db4c9607c99f\\\",\\r\\n \\\"activityId\\\": \\\"c7172e67-5da6-46b0-ac83-c1a5b5be4ded\\\",\\r\\n \\\"subActivityId\\\": \\\"a1170dba-d226-4772-89b6-9f45fb03d368\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d3feb915-cfb4-4185-8525-9185496fce8d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;16d19bd3-8d48-4dca-aa85-db4c9607c99f ARID=c7172e67-5da6-46b0-ac83-c1a5b5be4ded \\u003e GW.Http.CallContext/c7172e67-5da6-46b0-ac83-c1a5b5be4ded \\u003e GW.ExecuteQuery/d3feb915-cfb4-4185-8525-9185496fce8d \\u003e DN.FE.ExecuteQuery/a1170dba-d226-4772-89b6-9f45fb03d368)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:41:52.877]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:41:49.6902087Z);\\nlet _endTime = datetime(2026-08-24T21:41:49.6902091Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sdm9vzp4vv2s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:41:52.8452126Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;7ded6e3a-2d78-4f8a-bb3e-3fe5cff3f43c\\\",\\r\\n \\\"activityId\\\": \\\"9f24226b-f7d3-4eab-aea7-69e5e64814d8\\\",\\r\\n \\\"subActivityId\\\": \\\"2a442ad9-3953-4a76-a5d2-fa2895704a6b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7111592a-ec51-4e3b-862a-f171de9c1066\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7ded6e3a-2d78-4f8a-bb3e-3fe5cff3f43c ARID=9f24226b-f7d3-4eab-aea7-69e5e64814d8 \\u003e GW.Http.CallContext/9f24226b-f7d3-4eab-aea7-69e5e64814d8 \\u003e GW.ExecuteQuery/7111592a-ec51-4e3b-862a-f171de9c1066 \\u003e DN.FE.ExecuteQuery/2a442ad9-3953-4a76-a5d2-fa2895704a6b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:41:53.034]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:41:49.6902087Z);\\nlet _endTime = datetime(2026-08-24T21:41:49.6902091Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sdm9vzp4vv2s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:41:52.8452126Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;7ded6e3a-2d78-4f8a-bb3e-3fe5cff3f43c\\\",\\r\\n \\\"activityId\\\": \\\"9f24226b-f7d3-4eab-aea7-69e5e64814d8\\\",\\r\\n \\\"subActivityId\\\": \\\"2a442ad9-3953-4a76-a5d2-fa2895704a6b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7111592a-ec51-4e3b-862a-f171de9c1066\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7ded6e3a-2d78-4f8a-bb3e-3fe5cff3f43c ARID=9f24226b-f7d3-4eab-aea7-69e5e64814d8 \\u003e GW.Http.CallContext/9f24226b-f7d3-4eab-aea7-69e5e64814d8 \\u003e GW.ExecuteQuery/7111592a-ec51-4e3b-862a-f171de9c1066 \\u003e DN.FE.ExecuteQuery/2a442ad9-3953-4a76-a5d2-fa2895704a6b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 604.553s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001b12490\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:41:43.018]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:41:39.0159138Z);\\nlet _endTime = datetime(2026-08-24T21:41:39.0159143Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sdm9vzp4vv2s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:41:42.9938618Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;42b7fe0e-1ca7-4eb3-bd94-70b13c5f50a0\\\",\\r\\n \\\"activityId\\\": \\\"428709cb-5faa-460e-9fd0-19b365a38358\\\",\\r\\n \\\"subActivityId\\\": \\\"c6511ec0-fe87-45c4-8535-22ba47859a18\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"73e933ce-67eb-4635-9995-9c7ad9fa82a5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;42b7fe0e-1ca7-4eb3-bd94-70b13c5f50a0 ARID=428709cb-5faa-460e-9fd0-19b365a38358 \\u003e GW.Http.CallContext/428709cb-5faa-460e-9fd0-19b365a38358 \\u003e GW.ExecuteQuery/73e933ce-67eb-4635-9995-9c7ad9fa82a5 \\u003e DN.FE.ExecuteQuery/c6511ec0-fe87-45c4-8535-22ba47859a18)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:41:43.076]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:41:39.0159138Z);\\nlet _endTime = datetime(2026-08-24T21:41:39.0159143Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sdm9vzp4vv2s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:41:42.9938618Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;42b7fe0e-1ca7-4eb3-bd94-70b13c5f50a0\\\",\\r\\n \\\"activityId\\\": \\\"428709cb-5faa-460e-9fd0-19b365a38358\\\",\\r\\n \\\"subActivityId\\\": \\\"c6511ec0-fe87-45c4-8535-22ba47859a18\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"73e933ce-67eb-4635-9995-9c7ad9fa82a5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;42b7fe0e-1ca7-4eb3-bd94-70b13c5f50a0 ARID=428709cb-5faa-460e-9fd0-19b365a38358 \\u003e GW.Http.CallContext/428709cb-5faa-460e-9fd0-19b365a38358 \\u003e GW.ExecuteQuery/73e933ce-67eb-4635-9995-9c7ad9fa82a5 \\u003e DN.FE.ExecuteQuery/c6511ec0-fe87-45c4-8535-22ba47859a18)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:41:46.333]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:41:43.1465223Z);\\nlet _endTime = datetime(2026-08-24T21:41:43.1465227Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sdm9vzp4vv2s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:41:46.3064953Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;16d19bd3-8d48-4dca-aa85-db4c9607c99f\\\",\\r\\n \\\"activityId\\\": \\\"c7172e67-5da6-46b0-ac83-c1a5b5be4ded\\\",\\r\\n \\\"subActivityId\\\": \\\"a1170dba-d226-4772-89b6-9f45fb03d368\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d3feb915-cfb4-4185-8525-9185496fce8d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;16d19bd3-8d48-4dca-aa85-db4c9607c99f ARID=c7172e67-5da6-46b0-ac83-c1a5b5be4ded \\u003e GW.Http.CallContext/c7172e67-5da6-46b0-ac83-c1a5b5be4ded \\u003e GW.ExecuteQuery/d3feb915-cfb4-4185-8525-9185496fce8d \\u003e DN.FE.ExecuteQuery/a1170dba-d226-4772-89b6-9f45fb03d368)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:41:46.391]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:41:43.1465223Z);\\nlet _endTime = datetime(2026-08-24T21:41:43.1465227Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sdm9vzp4vv2s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:41:46.3064953Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;16d19bd3-8d48-4dca-aa85-db4c9607c99f\\\",\\r\\n \\\"activityId\\\": \\\"c7172e67-5da6-46b0-ac83-c1a5b5be4ded\\\",\\r\\n \\\"subActivityId\\\": \\\"a1170dba-d226-4772-89b6-9f45fb03d368\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d3feb915-cfb4-4185-8525-9185496fce8d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;16d19bd3-8d48-4dca-aa85-db4c9607c99f ARID=c7172e67-5da6-46b0-ac83-c1a5b5be4ded \\u003e GW.Http.CallContext/c7172e67-5da6-46b0-ac83-c1a5b5be4ded \\u003e GW.ExecuteQuery/d3feb915-cfb4-4185-8525-9185496fce8d \\u003e DN.FE.ExecuteQuery/a1170dba-d226-4772-89b6-9f45fb03d368)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:41:52.877]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:41:49.6902087Z);\\nlet _endTime = datetime(2026-08-24T21:41:49.6902091Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sdm9vzp4vv2s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:41:52.8452126Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;7ded6e3a-2d78-4f8a-bb3e-3fe5cff3f43c\\\",\\r\\n \\\"activityId\\\": \\\"9f24226b-f7d3-4eab-aea7-69e5e64814d8\\\",\\r\\n \\\"subActivityId\\\": \\\"2a442ad9-3953-4a76-a5d2-fa2895704a6b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7111592a-ec51-4e3b-862a-f171de9c1066\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7ded6e3a-2d78-4f8a-bb3e-3fe5cff3f43c ARID=9f24226b-f7d3-4eab-aea7-69e5e64814d8 \\u003e GW.Http.CallContext/9f24226b-f7d3-4eab-aea7-69e5e64814d8 \\u003e GW.ExecuteQuery/7111592a-ec51-4e3b-862a-f171de9c1066 \\u003e DN.FE.ExecuteQuery/2a442ad9-3953-4a76-a5d2-fa2895704a6b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:41:53.034]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:41:49.6902087Z);\\nlet _endTime = datetime(2026-08-24T21:41:49.6902091Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-sdm9vzp4vv2s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:41:52.8452126Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;7ded6e3a-2d78-4f8a-bb3e-3fe5cff3f43c\\\",\\r\\n \\\"activityId\\\": \\\"9f24226b-f7d3-4eab-aea7-69e5e64814d8\\\",\\r\\n \\\"subActivityId\\\": \\\"2a442ad9-3953-4a76-a5d2-fa2895704a6b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7111592a-ec51-4e3b-862a-f171de9c1066\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7ded6e3a-2d78-4f8a-bb3e-3fe5cff3f43c ARID=9f24226b-f7d3-4eab-aea7-69e5e64814d8 \\u003e GW.Http.CallContext/9f24226b-f7d3-4eab-aea7-69e5e64814d8 \\u003e GW.ExecuteQuery/7111592a-ec51-4e3b-862a-f171de9c1066 \\u003e DN.FE.ExecuteQuery/2a442ad9-3953-4a76-a5d2-fa2895704a6b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"c5b4e0422bdbb4a2a93df57e5cd162114b2b40c9b1dd487f267e3e5845b55a49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985911028912128","occurred_at":"2026-08-24T20:28:03Z","failed_at":"e2e","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985362346840064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6662,"pr_state":"open","pr_sha":"87c9d3ce24afa32ec6516cc621a3203652172097","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T20:25:41Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"07fba13c352d0df761c12de34205d2078ff8abec7581f20ad66cf57f5a712827","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985362346840064","occurred_at":"2026-08-24T20:25:41Z","failed_at":"e2e","signature_id":"176cc4504c2f048683d4e9fd8346ddda26590ba46f5fcff44ce3756a85f5565e","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 606.962s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0009b8890\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:33:39.927]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:33:34.7399088Z);\\nlet _endTime = datetime(2026-08-24T21:33:34.7399092Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8x5xzz8skd6l'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:33:39.9041454Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;3ee56433-bb81-45eb-b932-ee828be0698d\\\",\\r\\n \\\"activityId\\\": \\\"0b55016d-8c54-42c2-8cdc-8a328b8d6d7d\\\",\\r\\n \\\"subActivityId\\\": \\\"5fec2564-1b41-4e19-8b38-9f621cfecc7d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c71f0e3b-688d-473b-837f-87b8ee173b88\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;3ee56433-bb81-45eb-b932-ee828be0698d ARID=0b55016d-8c54-42c2-8cdc-8a328b8d6d7d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/861bdbb2-7dca-4bb1-a9cf-b7c3656731d7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/861bdbb2-7dca-4bb1-a9cf-b7c3656731d7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/c71f0e3b-688d-473b-837f-87b8ee173b88 \\u003e DN.FE.ExecuteQuery/5fec2564-1b41-4e19-8b38-9f621cfecc7d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:33:40.031]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:33:34.7399088Z);\\nlet _endTime = datetime(2026-08-24T21:33:34.7399092Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8x5xzz8skd6l'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:33:39.9041454Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;3ee56433-bb81-45eb-b932-ee828be0698d\\\",\\r\\n \\\"activityId\\\": \\\"0b55016d-8c54-42c2-8cdc-8a328b8d6d7d\\\",\\r\\n \\\"subActivityId\\\": \\\"5fec2564-1b41-4e19-8b38-9f621cfecc7d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c71f0e3b-688d-473b-837f-87b8ee173b88\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;3ee56433-bb81-45eb-b932-ee828be0698d ARID=0b55016d-8c54-42c2-8cdc-8a328b8d6d7d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/861bdbb2-7dca-4bb1-a9cf-b7c3656731d7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/861bdbb2-7dca-4bb1-a9cf-b7c3656731d7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/c71f0e3b-688d-473b-837f-87b8ee173b88 \\u003e DN.FE.ExecuteQuery/5fec2564-1b41-4e19-8b38-9f621cfecc7d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:33:45.871]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:33:40.0938878Z);\\nlet _endTime = datetime(2026-08-24T21:33:40.0938882Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8x5xzz8skd6l'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:33:45.8479456Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;96873d82-5ef0-43eb-a74d-45256ccc6d85\\\",\\r\\n \\\"activityId\\\": \\\"a7493440-b2f3-4e69-8824-c220555bf425\\\",\\r\\n \\\"subActivityId\\\": \\\"7f2f2c5e-0c26-4249-9c73-f463c8826d3d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"363a2ae3-d9d7-4e71-a139-922284c005f1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;96873d82-5ef0-43eb-a74d-45256ccc6d85 ARID=a7493440-b2f3-4e69-8824-c220555bf425 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/cbf872ee-2e17-4232-b13d-a7e05647474a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/cbf872ee-2e17-4232-b13d-a7e05647474a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/363a2ae3-d9d7-4e71-a139-922284c005f1 \\u003e DN.FE.ExecuteQuery/7f2f2c5e-0c26-4249-9c73-f463c8826d3d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:33:45.923]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:33:40.0938878Z);\\nlet _endTime = datetime(2026-08-24T21:33:40.0938882Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8x5xzz8skd6l'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:33:45.8479456Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;96873d82-5ef0-43eb-a74d-45256ccc6d85\\\",\\r\\n \\\"activityId\\\": \\\"a7493440-b2f3-4e69-8824-c220555bf425\\\",\\r\\n \\\"subActivityId\\\": \\\"7f2f2c5e-0c26-4249-9c73-f463c8826d3d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"363a2ae3-d9d7-4e71-a139-922284c005f1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;96873d82-5ef0-43eb-a74d-45256ccc6d85 ARID=a7493440-b2f3-4e69-8824-c220555bf425 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/cbf872ee-2e17-4232-b13d-a7e05647474a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/cbf872ee-2e17-4232-b13d-a7e05647474a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/363a2ae3-d9d7-4e71-a139-922284c005f1 \\u003e DN.FE.ExecuteQuery/7f2f2c5e-0c26-4249-9c73-f463c8826d3d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:33:53.707]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:33:50.1150311Z);\\nlet _endTime = datetime(2026-08-24T21:33:50.1150315Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8x5xzz8skd6l'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:33:53.6861063Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2008e7e2-f66d-4bba-ab0c-fa9c9cc8792b\\\",\\r\\n \\\"activityId\\\": \\\"2e144aa9-1441-4208-9ef8-1602c42e9f13\\\",\\r\\n \\\"subActivityId\\\": \\\"8a6c32f3-9065-40d0-b597-c44fd5dbbac2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"381df9bd-a94b-477c-879b-22493a4271de\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2008e7e2-f66d-4bba-ab0c-fa9c9cc8792b ARID=2e144aa9-1441-4208-9ef8-1602c42e9f13 \\u003e GW.Http.CallContext/2e144aa9-1441-4208-9ef8-1602c42e9f13 \\u003e GW.ExecuteQuery/381df9bd-a94b-477c-879b-22493a4271de \\u003e DN.FE.ExecuteQuery/8a6c32f3-9065-40d0-b597-c44fd5dbbac2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:33:53.799]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:33:50.1150311Z);\\nlet _endTime = datetime(2026-08-24T21:33:50.1150315Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8x5xzz8skd6l'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:33:53.6861063Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2008e7e2-f66d-4bba-ab0c-fa9c9cc8792b\\\",\\r\\n \\\"activityId\\\": \\\"2e144aa9-1441-4208-9ef8-1602c42e9f13\\\",\\r\\n \\\"subActivityId\\\": \\\"8a6c32f3-9065-40d0-b597-c44fd5dbbac2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"381df9bd-a94b-477c-879b-22493a4271de\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2008e7e2-f66d-4bba-ab0c-fa9c9cc8792b ARID=2e144aa9-1441-4208-9ef8-1602c42e9f13 \\u003e GW.Http.CallContext/2e144aa9-1441-4208-9ef8-1602c42e9f13 \\u003e GW.ExecuteQuery/381df9bd-a94b-477c-879b-22493a4271de \\u003e DN.FE.ExecuteQuery/8a6c32f3-9065-40d0-b597-c44fd5dbbac2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 606.962s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0009b8890\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:33:39.927]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:33:34.7399088Z);\\nlet _endTime = datetime(2026-08-24T21:33:34.7399092Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8x5xzz8skd6l'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:33:39.9041454Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;3ee56433-bb81-45eb-b932-ee828be0698d\\\",\\r\\n \\\"activityId\\\": \\\"0b55016d-8c54-42c2-8cdc-8a328b8d6d7d\\\",\\r\\n \\\"subActivityId\\\": \\\"5fec2564-1b41-4e19-8b38-9f621cfecc7d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c71f0e3b-688d-473b-837f-87b8ee173b88\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;3ee56433-bb81-45eb-b932-ee828be0698d ARID=0b55016d-8c54-42c2-8cdc-8a328b8d6d7d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/861bdbb2-7dca-4bb1-a9cf-b7c3656731d7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/861bdbb2-7dca-4bb1-a9cf-b7c3656731d7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/c71f0e3b-688d-473b-837f-87b8ee173b88 \\u003e DN.FE.ExecuteQuery/5fec2564-1b41-4e19-8b38-9f621cfecc7d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:33:40.031]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:33:34.7399088Z);\\nlet _endTime = datetime(2026-08-24T21:33:34.7399092Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8x5xzz8skd6l'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:33:39.9041454Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;3ee56433-bb81-45eb-b932-ee828be0698d\\\",\\r\\n \\\"activityId\\\": \\\"0b55016d-8c54-42c2-8cdc-8a328b8d6d7d\\\",\\r\\n \\\"subActivityId\\\": \\\"5fec2564-1b41-4e19-8b38-9f621cfecc7d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c71f0e3b-688d-473b-837f-87b8ee173b88\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;3ee56433-bb81-45eb-b932-ee828be0698d ARID=0b55016d-8c54-42c2-8cdc-8a328b8d6d7d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/861bdbb2-7dca-4bb1-a9cf-b7c3656731d7 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/861bdbb2-7dca-4bb1-a9cf-b7c3656731d7 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/c71f0e3b-688d-473b-837f-87b8ee173b88 \\u003e DN.FE.ExecuteQuery/5fec2564-1b41-4e19-8b38-9f621cfecc7d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:33:45.871]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:33:40.0938878Z);\\nlet _endTime = datetime(2026-08-24T21:33:40.0938882Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8x5xzz8skd6l'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:33:45.8479456Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;96873d82-5ef0-43eb-a74d-45256ccc6d85\\\",\\r\\n \\\"activityId\\\": \\\"a7493440-b2f3-4e69-8824-c220555bf425\\\",\\r\\n \\\"subActivityId\\\": \\\"7f2f2c5e-0c26-4249-9c73-f463c8826d3d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"363a2ae3-d9d7-4e71-a139-922284c005f1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;96873d82-5ef0-43eb-a74d-45256ccc6d85 ARID=a7493440-b2f3-4e69-8824-c220555bf425 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/cbf872ee-2e17-4232-b13d-a7e05647474a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/cbf872ee-2e17-4232-b13d-a7e05647474a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/363a2ae3-d9d7-4e71-a139-922284c005f1 \\u003e DN.FE.ExecuteQuery/7f2f2c5e-0c26-4249-9c73-f463c8826d3d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:33:45.923]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:33:40.0938878Z);\\nlet _endTime = datetime(2026-08-24T21:33:40.0938882Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8x5xzz8skd6l'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:33:45.8479456Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;96873d82-5ef0-43eb-a74d-45256ccc6d85\\\",\\r\\n \\\"activityId\\\": \\\"a7493440-b2f3-4e69-8824-c220555bf425\\\",\\r\\n \\\"subActivityId\\\": \\\"7f2f2c5e-0c26-4249-9c73-f463c8826d3d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"363a2ae3-d9d7-4e71-a139-922284c005f1\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;96873d82-5ef0-43eb-a74d-45256ccc6d85 ARID=a7493440-b2f3-4e69-8824-c220555bf425 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/cbf872ee-2e17-4232-b13d-a7e05647474a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/cbf872ee-2e17-4232-b13d-a7e05647474a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/363a2ae3-d9d7-4e71-a139-922284c005f1 \\u003e DN.FE.ExecuteQuery/7f2f2c5e-0c26-4249-9c73-f463c8826d3d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:33:53.707]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:33:50.1150311Z);\\nlet _endTime = datetime(2026-08-24T21:33:50.1150315Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8x5xzz8skd6l'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:33:53.6861063Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2008e7e2-f66d-4bba-ab0c-fa9c9cc8792b\\\",\\r\\n \\\"activityId\\\": \\\"2e144aa9-1441-4208-9ef8-1602c42e9f13\\\",\\r\\n \\\"subActivityId\\\": \\\"8a6c32f3-9065-40d0-b597-c44fd5dbbac2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"381df9bd-a94b-477c-879b-22493a4271de\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2008e7e2-f66d-4bba-ab0c-fa9c9cc8792b ARID=2e144aa9-1441-4208-9ef8-1602c42e9f13 \\u003e GW.Http.CallContext/2e144aa9-1441-4208-9ef8-1602c42e9f13 \\u003e GW.ExecuteQuery/381df9bd-a94b-477c-879b-22493a4271de \\u003e DN.FE.ExecuteQuery/8a6c32f3-9065-40d0-b597-c44fd5dbbac2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:33:53.799]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:33:50.1150311Z);\\nlet _endTime = datetime(2026-08-24T21:33:50.1150315Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8x5xzz8skd6l'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:33:53.6861063Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2008e7e2-f66d-4bba-ab0c-fa9c9cc8792b\\\",\\r\\n \\\"activityId\\\": \\\"2e144aa9-1441-4208-9ef8-1602c42e9f13\\\",\\r\\n \\\"subActivityId\\\": \\\"8a6c32f3-9065-40d0-b597-c44fd5dbbac2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"381df9bd-a94b-477c-879b-22493a4271de\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2008e7e2-f66d-4bba-ab0c-fa9c9cc8792b ARID=2e144aa9-1441-4208-9ef8-1602c42e9f13 \\u003e GW.Http.CallContext/2e144aa9-1441-4208-9ef8-1602c42e9f13 \\u003e GW.ExecuteQuery/381df9bd-a94b-477c-879b-22493a4271de \\u003e DN.FE.ExecuteQuery/8a6c32f3-9065-40d0-b597-c44fd5dbbac2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6641,"pr_state":"open","pr_sha":"2a2bf6c0e8f39a611bcc3192554e764493be1383","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T20:20:43Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"17c6c0e7a9c44d0e0b3cb94052e4483a9d626b2549649480d31b7e537346236b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","failed_at":"e2e","signature_id":"7451289a87442818c6daa03144d0fb9efebafa88d3687d31f99e8439ca741e97","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a7afc0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-qgs92k2n68xw\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f8d27b3b-7241-49b3-bf1d-66bcf6fbfc8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f8d27b3b-7241-49b3-bf1d-66bcf6fbfc8a\",\n \"name\": \"f8d27b3b-7241-49b3-bf1d-66bcf6fbfc8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:11:59.82493651Z\",\n \"endTime\": \"2026-08-24T21:31:07.397499058Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a7afc0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-qgs92k2n68xw\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f8d27b3b-7241-49b3-bf1d-66bcf6fbfc8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f8d27b3b-7241-49b3-bf1d-66bcf6fbfc8a\",\n \"name\": \"f8d27b3b-7241-49b3-bf1d-66bcf6fbfc8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:11:59.82493651Z\",\n \"endTime\": \"2026-08-24T21:31:07.397499058Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"41886320012e37602e6d85e79b963add62efafd3c73dd81a37b63fc23a694e3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","failed_at":"alert","signature_id":"05c28635da15b6abb7fa52304cc82cedf7070115a67070cb7058c7bc17ceed55","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T21:24:11Z\n Ended: 2026-08-24T21:41:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8136960-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd6bajvc8sdpu7eqok3bv8u764jvvj5-cilium-cluster\", pod=\"router-67b4cfd85c-jvk8t\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd6bajvc8sdpu7eqok3bv8u764jvvj5-cilium-cluster/router-67b4cfd85c-jvk8t has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T21:24:11Z\n Ended: 2026-08-24T21:44:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8136960-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd6bqj0fesvbh7g59o57oi9i09o3vve-agg-cluster\", pod=\"router-7cf7b968fd-ssk4r\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd6bqj0fesvbh7g59o57oi9i09o3vve-agg-cluster/router-7cf7b968fd-ssk4r has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"42e91712b5243ecbe10c8ce244c4d3fa76c849a0a12fc14e9718178e9776a97f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","failed_at":"e2e","signature_id":"fb242b6b09da99dc7298c2563fdd22c18c2c33f65971895aaf866ce3967b98d3","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012a4ea0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-26mdm7dwkvm7\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fea58607-6b10-431a-a2a1-cc0c116e10b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fea58607-6b10-431a-a2a1-cc0c116e10b9\",\n \"name\": \"fea58607-6b10-431a-a2a1-cc0c116e10b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:12:09.791388432Z\",\n \"endTime\": \"2026-08-24T21:31:17.457093893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012a4ea0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-26mdm7dwkvm7\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fea58607-6b10-431a-a2a1-cc0c116e10b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fea58607-6b10-431a-a2a1-cc0c116e10b9\",\n \"name\": \"fea58607-6b10-431a-a2a1-cc0c116e10b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:12:09.791388432Z\",\n \"endTime\": \"2026-08-24T21:31:17.457093893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5a762dca641766745080da7d770a778bf9dbce9c0749a5182a3e1a5df93568dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","failed_at":"e2e","signature_id":"913731872ab076d2435270c0fedcc39b2d88a5ff36047161ca4eabe2b648b67d","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 619.218s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0010003e0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:37:40.862]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:37:37.2998276Z);\\nlet _endTime = datetime(2026-08-24T21:37:37.2998280Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-k8dn8g4r8ltr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:37:40.8395761Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a7f2f448-ead0-4bf7-a0e6-5fcabf17d38a\\\",\\r\\n \\\"activityId\\\": \\\"e9aa0456-31d3-4bdf-a2f9-0cf51d841f5e\\\",\\r\\n \\\"subActivityId\\\": \\\"43a214bb-e28a-4d77-8a51-8eb215a63034\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"99f96206-4dd7-46c4-a9b8-10491eab64a3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a7f2f448-ead0-4bf7-a0e6-5fcabf17d38a ARID=e9aa0456-31d3-4bdf-a2f9-0cf51d841f5e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/07c89248-2432-450a-a970-6507b073f540 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/07c89248-2432-450a-a970-6507b073f540 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/99f96206-4dd7-46c4-a9b8-10491eab64a3 \\u003e DN.FE.ExecuteQuery/43a214bb-e28a-4d77-8a51-8eb215a63034)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:37:40.916]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:37:37.2998276Z);\\nlet _endTime = datetime(2026-08-24T21:37:37.2998280Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-k8dn8g4r8ltr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:37:40.8395761Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a7f2f448-ead0-4bf7-a0e6-5fcabf17d38a\\\",\\r\\n \\\"activityId\\\": \\\"e9aa0456-31d3-4bdf-a2f9-0cf51d841f5e\\\",\\r\\n \\\"subActivityId\\\": \\\"43a214bb-e28a-4d77-8a51-8eb215a63034\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"99f96206-4dd7-46c4-a9b8-10491eab64a3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a7f2f448-ead0-4bf7-a0e6-5fcabf17d38a ARID=e9aa0456-31d3-4bdf-a2f9-0cf51d841f5e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/07c89248-2432-450a-a970-6507b073f540 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/07c89248-2432-450a-a970-6507b073f540 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/99f96206-4dd7-46c4-a9b8-10491eab64a3 \\u003e DN.FE.ExecuteQuery/43a214bb-e28a-4d77-8a51-8eb215a63034)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:37:44.776]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:37:40.9782699Z);\\nlet _endTime = datetime(2026-08-24T21:37:40.9782702Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-k8dn8g4r8ltr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:37:44.4496384Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a6ab5682-ab15-4ffc-8f2c-903f04846256\\\",\\r\\n \\\"activityId\\\": \\\"41da04b9-cdd8-4f1b-aa71-799246592373\\\",\\r\\n \\\"subActivityId\\\": \\\"8180892d-1919-40e9-ab32-111b1441a675\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f68272df-da5c-49db-9d06-dcd9340019b6\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a6ab5682-ab15-4ffc-8f2c-903f04846256 ARID=41da04b9-cdd8-4f1b-aa71-799246592373 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/375d1995-37d2-46ea-a910-0cf7e62e840b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/375d1995-37d2-46ea-a910-0cf7e62e840b \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f68272df-da5c-49db-9d06-dcd9340019b6 \\u003e DN.FE.ExecuteQuery/8180892d-1919-40e9-ab32-111b1441a675)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:37:45.273]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:37:40.9782699Z);\\nlet _endTime = datetime(2026-08-24T21:37:40.9782702Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-k8dn8g4r8ltr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:37:44.4496384Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a6ab5682-ab15-4ffc-8f2c-903f04846256\\\",\\r\\n \\\"activityId\\\": \\\"41da04b9-cdd8-4f1b-aa71-799246592373\\\",\\r\\n \\\"subActivityId\\\": \\\"8180892d-1919-40e9-ab32-111b1441a675\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f68272df-da5c-49db-9d06-dcd9340019b6\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a6ab5682-ab15-4ffc-8f2c-903f04846256 ARID=41da04b9-cdd8-4f1b-aa71-799246592373 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/375d1995-37d2-46ea-a910-0cf7e62e840b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/375d1995-37d2-46ea-a910-0cf7e62e840b \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f68272df-da5c-49db-9d06-dcd9340019b6 \\u003e DN.FE.ExecuteQuery/8180892d-1919-40e9-ab32-111b1441a675)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:37:52.048]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:37:48.7440398Z);\\nlet _endTime = datetime(2026-08-24T21:37:48.7440401Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-k8dn8g4r8ltr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:37:52.0276361Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2d45695e-bd83-4f2f-ad2d-1f2db8c593b0\\\",\\r\\n \\\"activityId\\\": \\\"0e05d86a-4495-4335-ba26-b5db97c01353\\\",\\r\\n \\\"subActivityId\\\": \\\"2afcfb00-a02e-49a7-9141-8b2e3bea0987\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6cc0f216-19d3-43c5-b12c-5fdd754b74bf\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2d45695e-bd83-4f2f-ad2d-1f2db8c593b0 ARID=0e05d86a-4495-4335-ba26-b5db97c01353 \\u003e GW.Http.CallContext/0e05d86a-4495-4335-ba26-b5db97c01353 \\u003e GW.ExecuteQuery/6cc0f216-19d3-43c5-b12c-5fdd754b74bf \\u003e DN.FE.ExecuteQuery/2afcfb00-a02e-49a7-9141-8b2e3bea0987)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:37:52.603]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:37:48.7440398Z);\\nlet _endTime = datetime(2026-08-24T21:37:48.7440401Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-k8dn8g4r8ltr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:37:52.0276361Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2d45695e-bd83-4f2f-ad2d-1f2db8c593b0\\\",\\r\\n \\\"activityId\\\": \\\"0e05d86a-4495-4335-ba26-b5db97c01353\\\",\\r\\n \\\"subActivityId\\\": \\\"2afcfb00-a02e-49a7-9141-8b2e3bea0987\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6cc0f216-19d3-43c5-b12c-5fdd754b74bf\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2d45695e-bd83-4f2f-ad2d-1f2db8c593b0 ARID=0e05d86a-4495-4335-ba26-b5db97c01353 \\u003e GW.Http.CallContext/0e05d86a-4495-4335-ba26-b5db97c01353 \\u003e GW.ExecuteQuery/6cc0f216-19d3-43c5-b12c-5fdd754b74bf \\u003e DN.FE.ExecuteQuery/2afcfb00-a02e-49a7-9141-8b2e3bea0987)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 619.218s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0010003e0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:37:40.862]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:37:37.2998276Z);\\nlet _endTime = datetime(2026-08-24T21:37:37.2998280Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-k8dn8g4r8ltr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:37:40.8395761Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a7f2f448-ead0-4bf7-a0e6-5fcabf17d38a\\\",\\r\\n \\\"activityId\\\": \\\"e9aa0456-31d3-4bdf-a2f9-0cf51d841f5e\\\",\\r\\n \\\"subActivityId\\\": \\\"43a214bb-e28a-4d77-8a51-8eb215a63034\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"99f96206-4dd7-46c4-a9b8-10491eab64a3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a7f2f448-ead0-4bf7-a0e6-5fcabf17d38a ARID=e9aa0456-31d3-4bdf-a2f9-0cf51d841f5e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/07c89248-2432-450a-a970-6507b073f540 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/07c89248-2432-450a-a970-6507b073f540 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/99f96206-4dd7-46c4-a9b8-10491eab64a3 \\u003e DN.FE.ExecuteQuery/43a214bb-e28a-4d77-8a51-8eb215a63034)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:37:40.916]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:37:37.2998276Z);\\nlet _endTime = datetime(2026-08-24T21:37:37.2998280Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-k8dn8g4r8ltr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:37:40.8395761Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a7f2f448-ead0-4bf7-a0e6-5fcabf17d38a\\\",\\r\\n \\\"activityId\\\": \\\"e9aa0456-31d3-4bdf-a2f9-0cf51d841f5e\\\",\\r\\n \\\"subActivityId\\\": \\\"43a214bb-e28a-4d77-8a51-8eb215a63034\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"99f96206-4dd7-46c4-a9b8-10491eab64a3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a7f2f448-ead0-4bf7-a0e6-5fcabf17d38a ARID=e9aa0456-31d3-4bdf-a2f9-0cf51d841f5e \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/07c89248-2432-450a-a970-6507b073f540 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/07c89248-2432-450a-a970-6507b073f540 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/99f96206-4dd7-46c4-a9b8-10491eab64a3 \\u003e DN.FE.ExecuteQuery/43a214bb-e28a-4d77-8a51-8eb215a63034)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:37:44.776]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:37:40.9782699Z);\\nlet _endTime = datetime(2026-08-24T21:37:40.9782702Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-k8dn8g4r8ltr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:37:44.4496384Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a6ab5682-ab15-4ffc-8f2c-903f04846256\\\",\\r\\n \\\"activityId\\\": \\\"41da04b9-cdd8-4f1b-aa71-799246592373\\\",\\r\\n \\\"subActivityId\\\": \\\"8180892d-1919-40e9-ab32-111b1441a675\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f68272df-da5c-49db-9d06-dcd9340019b6\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a6ab5682-ab15-4ffc-8f2c-903f04846256 ARID=41da04b9-cdd8-4f1b-aa71-799246592373 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/375d1995-37d2-46ea-a910-0cf7e62e840b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/375d1995-37d2-46ea-a910-0cf7e62e840b \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f68272df-da5c-49db-9d06-dcd9340019b6 \\u003e DN.FE.ExecuteQuery/8180892d-1919-40e9-ab32-111b1441a675)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:37:45.273]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:37:40.9782699Z);\\nlet _endTime = datetime(2026-08-24T21:37:40.9782702Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-k8dn8g4r8ltr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:37:44.4496384Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a6ab5682-ab15-4ffc-8f2c-903f04846256\\\",\\r\\n \\\"activityId\\\": \\\"41da04b9-cdd8-4f1b-aa71-799246592373\\\",\\r\\n \\\"subActivityId\\\": \\\"8180892d-1919-40e9-ab32-111b1441a675\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f68272df-da5c-49db-9d06-dcd9340019b6\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a6ab5682-ab15-4ffc-8f2c-903f04846256 ARID=41da04b9-cdd8-4f1b-aa71-799246592373 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/375d1995-37d2-46ea-a910-0cf7e62e840b \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/375d1995-37d2-46ea-a910-0cf7e62e840b \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f68272df-da5c-49db-9d06-dcd9340019b6 \\u003e DN.FE.ExecuteQuery/8180892d-1919-40e9-ab32-111b1441a675)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:37:52.048]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:37:48.7440398Z);\\nlet _endTime = datetime(2026-08-24T21:37:48.7440401Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-k8dn8g4r8ltr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:37:52.0276361Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2d45695e-bd83-4f2f-ad2d-1f2db8c593b0\\\",\\r\\n \\\"activityId\\\": \\\"0e05d86a-4495-4335-ba26-b5db97c01353\\\",\\r\\n \\\"subActivityId\\\": \\\"2afcfb00-a02e-49a7-9141-8b2e3bea0987\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6cc0f216-19d3-43c5-b12c-5fdd754b74bf\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2d45695e-bd83-4f2f-ad2d-1f2db8c593b0 ARID=0e05d86a-4495-4335-ba26-b5db97c01353 \\u003e GW.Http.CallContext/0e05d86a-4495-4335-ba26-b5db97c01353 \\u003e GW.ExecuteQuery/6cc0f216-19d3-43c5-b12c-5fdd754b74bf \\u003e DN.FE.ExecuteQuery/2afcfb00-a02e-49a7-9141-8b2e3bea0987)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:37:52.603]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:37:48.7440398Z);\\nlet _endTime = datetime(2026-08-24T21:37:48.7440401Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-k8dn8g4r8ltr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:37:52.0276361Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2d45695e-bd83-4f2f-ad2d-1f2db8c593b0\\\",\\r\\n \\\"activityId\\\": \\\"0e05d86a-4495-4335-ba26-b5db97c01353\\\",\\r\\n \\\"subActivityId\\\": \\\"2afcfb00-a02e-49a7-9141-8b2e3bea0987\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6cc0f216-19d3-43c5-b12c-5fdd754b74bf\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2d45695e-bd83-4f2f-ad2d-1f2db8c593b0 ARID=0e05d86a-4495-4335-ba26-b5db97c01353 \\u003e GW.Http.CallContext/0e05d86a-4495-4335-ba26-b5db97c01353 \\u003e GW.ExecuteQuery/6cc0f216-19d3-43c5-b12c-5fdd754b74bf \\u003e DN.FE.ExecuteQuery/2afcfb00-a02e-49a7-9141-8b2e3bea0987)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"e54a3e6c5065c7942b54d91162708587f80328221571048cd3f81da88ee775c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","failed_at":"e2e","signature_id":"ab3b51419f8536317834b49d26ead032c3d2df538686a40306860f09479fc7a4","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00133a200\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qp6jkgrm7jtk\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d814403-8a1a-482d-afa5-63e68ef4ed09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d814403-8a1a-482d-afa5-63e68ef4ed09\",\n \"name\": \"7d814403-8a1a-482d-afa5-63e68ef4ed09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:13:13.654576687Z\",\n \"endTime\": \"2026-08-24T21:32:17.358471293Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00133a200\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qp6jkgrm7jtk\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d814403-8a1a-482d-afa5-63e68ef4ed09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d814403-8a1a-482d-afa5-63e68ef4ed09\",\n \"name\": \"7d814403-8a1a-482d-afa5-63e68ef4ed09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T21:13:13.654576687Z\",\n \"endTime\": \"2026-08-24T21:32:17.358471293Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091978207036182528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6232,"pr_state":"open","pr_sha":"cbb67883add3cc85dd65c43e81a6cb584dbf8021","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T19:57:24Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"79bac32224572b60430cedf38e17bcf46c6103961678c3e7da7015efcfe21dde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091978207036182528","occurred_at":"2026-08-24T19:57:24Z","failed_at":"e2e","signature_id":"f663b43ed0edc7e9f34c5b7d6058cf02871c4b2cbf7c46200eb2800bb395c1fc","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 615.524s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0007d2820\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[20:53:53.225]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:53:49.8410583Z);\\nlet _endTime = datetime(2026-08-24T20:53:49.8410588Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bzlbgl4sq6l2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:53:53.2004335Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0bfeb491-43ef-465f-bbbe-a0328d62f533\\\",\\r\\n \\\"activityId\\\": \\\"b79f0cec-39d0-4d66-a2a8-6c17ae5a7fcf\\\",\\r\\n \\\"subActivityId\\\": \\\"13d8a775-b3f1-4baa-9a2d-494dfc0be1b2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9786189c-ea06-4d5e-a5cc-f8cf3be8aa0d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0bfeb491-43ef-465f-bbbe-a0328d62f533 ARID=b79f0cec-39d0-4d66-a2a8-6c17ae5a7fcf \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e9ea8ca-6016-4ebf-b108-9558f32002d5 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e9ea8ca-6016-4ebf-b108-9558f32002d5 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9786189c-ea06-4d5e-a5cc-f8cf3be8aa0d \\u003e DN.FE.ExecuteQuery/13d8a775-b3f1-4baa-9a2d-494dfc0be1b2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:53:53.288]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:53:49.8410583Z);\\nlet _endTime = datetime(2026-08-24T20:53:49.8410588Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bzlbgl4sq6l2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:53:53.2004335Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0bfeb491-43ef-465f-bbbe-a0328d62f533\\\",\\r\\n \\\"activityId\\\": \\\"b79f0cec-39d0-4d66-a2a8-6c17ae5a7fcf\\\",\\r\\n \\\"subActivityId\\\": \\\"13d8a775-b3f1-4baa-9a2d-494dfc0be1b2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9786189c-ea06-4d5e-a5cc-f8cf3be8aa0d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0bfeb491-43ef-465f-bbbe-a0328d62f533 ARID=b79f0cec-39d0-4d66-a2a8-6c17ae5a7fcf \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e9ea8ca-6016-4ebf-b108-9558f32002d5 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e9ea8ca-6016-4ebf-b108-9558f32002d5 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9786189c-ea06-4d5e-a5cc-f8cf3be8aa0d \\u003e DN.FE.ExecuteQuery/13d8a775-b3f1-4baa-9a2d-494dfc0be1b2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[20:53:56.484]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:53:53.3762355Z);\\nlet _endTime = datetime(2026-08-24T20:53:53.3762363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bzlbgl4sq6l2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:53:56.4596189Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;50c48c80-ca0d-4a4d-870a-71c254e75198\\\",\\r\\n \\\"activityId\\\": \\\"95683e67-f053-4393-99b1-71151c466434\\\",\\r\\n \\\"subActivityId\\\": \\\"661413b7-6319-4d4e-84ca-2ab054915379\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6ac9b874-fb05-4d4b-806e-e7d9f8496026\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;50c48c80-ca0d-4a4d-870a-71c254e75198 ARID=95683e67-f053-4393-99b1-71151c466434 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f3695e3-ec59-4fef-9596-926ec623c669 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f3695e3-ec59-4fef-9596-926ec623c669 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6ac9b874-fb05-4d4b-806e-e7d9f8496026 \\u003e DN.FE.ExecuteQuery/661413b7-6319-4d4e-84ca-2ab054915379)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:53:56.541]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:53:53.3762355Z);\\nlet _endTime = datetime(2026-08-24T20:53:53.3762363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bzlbgl4sq6l2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:53:56.4596189Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;50c48c80-ca0d-4a4d-870a-71c254e75198\\\",\\r\\n \\\"activityId\\\": \\\"95683e67-f053-4393-99b1-71151c466434\\\",\\r\\n \\\"subActivityId\\\": \\\"661413b7-6319-4d4e-84ca-2ab054915379\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6ac9b874-fb05-4d4b-806e-e7d9f8496026\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;50c48c80-ca0d-4a4d-870a-71c254e75198 ARID=95683e67-f053-4393-99b1-71151c466434 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f3695e3-ec59-4fef-9596-926ec623c669 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f3695e3-ec59-4fef-9596-926ec623c669 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6ac9b874-fb05-4d4b-806e-e7d9f8496026 \\u003e DN.FE.ExecuteQuery/661413b7-6319-4d4e-84ca-2ab054915379)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[20:54:03.095]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:53:59.8371044Z);\\nlet _endTime = datetime(2026-08-24T20:53:59.8371051Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bzlbgl4sq6l2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:54:03.0699661Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;535f349f-0f02-4848-ad54-a4b9b368c05a\\\",\\r\\n \\\"activityId\\\": \\\"b8e19f30-0aeb-4bae-aa9a-2994c97ca8e2\\\",\\r\\n \\\"subActivityId\\\": \\\"c7079a92-9d3f-43af-ae10-33c5d509b1db\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"fe309183-9dd8-40b4-8fca-7e5ee982bcdd\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;535f349f-0f02-4848-ad54-a4b9b368c05a ARID=b8e19f30-0aeb-4bae-aa9a-2994c97ca8e2 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/30e8047d-b899-4df8-96bf-7f67a7ea3a80 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/30e8047d-b899-4df8-96bf-7f67a7ea3a80 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/fe309183-9dd8-40b4-8fca-7e5ee982bcdd \\u003e DN.FE.ExecuteQuery/c7079a92-9d3f-43af-ae10-33c5d509b1db)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:54:03.206]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:53:59.8371044Z);\\nlet _endTime = datetime(2026-08-24T20:53:59.8371051Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bzlbgl4sq6l2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:54:03.0699661Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;535f349f-0f02-4848-ad54-a4b9b368c05a\\\",\\r\\n \\\"activityId\\\": \\\"b8e19f30-0aeb-4bae-aa9a-2994c97ca8e2\\\",\\r\\n \\\"subActivityId\\\": \\\"c7079a92-9d3f-43af-ae10-33c5d509b1db\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"fe309183-9dd8-40b4-8fca-7e5ee982bcdd\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;535f349f-0f02-4848-ad54-a4b9b368c05a ARID=b8e19f30-0aeb-4bae-aa9a-2994c97ca8e2 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/30e8047d-b899-4df8-96bf-7f67a7ea3a80 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/30e8047d-b899-4df8-96bf-7f67a7ea3a80 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/fe309183-9dd8-40b4-8fca-7e5ee982bcdd \\u003e DN.FE.ExecuteQuery/c7079a92-9d3f-43af-ae10-33c5d509b1db)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 615.524s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0007d2820\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[20:53:53.225]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:53:49.8410583Z);\\nlet _endTime = datetime(2026-08-24T20:53:49.8410588Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bzlbgl4sq6l2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:53:53.2004335Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0bfeb491-43ef-465f-bbbe-a0328d62f533\\\",\\r\\n \\\"activityId\\\": \\\"b79f0cec-39d0-4d66-a2a8-6c17ae5a7fcf\\\",\\r\\n \\\"subActivityId\\\": \\\"13d8a775-b3f1-4baa-9a2d-494dfc0be1b2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9786189c-ea06-4d5e-a5cc-f8cf3be8aa0d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0bfeb491-43ef-465f-bbbe-a0328d62f533 ARID=b79f0cec-39d0-4d66-a2a8-6c17ae5a7fcf \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e9ea8ca-6016-4ebf-b108-9558f32002d5 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e9ea8ca-6016-4ebf-b108-9558f32002d5 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9786189c-ea06-4d5e-a5cc-f8cf3be8aa0d \\u003e DN.FE.ExecuteQuery/13d8a775-b3f1-4baa-9a2d-494dfc0be1b2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:53:53.288]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:53:49.8410583Z);\\nlet _endTime = datetime(2026-08-24T20:53:49.8410588Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bzlbgl4sq6l2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:53:53.2004335Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0bfeb491-43ef-465f-bbbe-a0328d62f533\\\",\\r\\n \\\"activityId\\\": \\\"b79f0cec-39d0-4d66-a2a8-6c17ae5a7fcf\\\",\\r\\n \\\"subActivityId\\\": \\\"13d8a775-b3f1-4baa-9a2d-494dfc0be1b2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9786189c-ea06-4d5e-a5cc-f8cf3be8aa0d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0bfeb491-43ef-465f-bbbe-a0328d62f533 ARID=b79f0cec-39d0-4d66-a2a8-6c17ae5a7fcf \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e9ea8ca-6016-4ebf-b108-9558f32002d5 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e9ea8ca-6016-4ebf-b108-9558f32002d5 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9786189c-ea06-4d5e-a5cc-f8cf3be8aa0d \\u003e DN.FE.ExecuteQuery/13d8a775-b3f1-4baa-9a2d-494dfc0be1b2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[20:53:56.484]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:53:53.3762355Z);\\nlet _endTime = datetime(2026-08-24T20:53:53.3762363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bzlbgl4sq6l2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:53:56.4596189Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;50c48c80-ca0d-4a4d-870a-71c254e75198\\\",\\r\\n \\\"activityId\\\": \\\"95683e67-f053-4393-99b1-71151c466434\\\",\\r\\n \\\"subActivityId\\\": \\\"661413b7-6319-4d4e-84ca-2ab054915379\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6ac9b874-fb05-4d4b-806e-e7d9f8496026\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;50c48c80-ca0d-4a4d-870a-71c254e75198 ARID=95683e67-f053-4393-99b1-71151c466434 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f3695e3-ec59-4fef-9596-926ec623c669 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f3695e3-ec59-4fef-9596-926ec623c669 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6ac9b874-fb05-4d4b-806e-e7d9f8496026 \\u003e DN.FE.ExecuteQuery/661413b7-6319-4d4e-84ca-2ab054915379)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:53:56.541]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:53:53.3762355Z);\\nlet _endTime = datetime(2026-08-24T20:53:53.3762363Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bzlbgl4sq6l2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:53:56.4596189Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;50c48c80-ca0d-4a4d-870a-71c254e75198\\\",\\r\\n \\\"activityId\\\": \\\"95683e67-f053-4393-99b1-71151c466434\\\",\\r\\n \\\"subActivityId\\\": \\\"661413b7-6319-4d4e-84ca-2ab054915379\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6ac9b874-fb05-4d4b-806e-e7d9f8496026\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;50c48c80-ca0d-4a4d-870a-71c254e75198 ARID=95683e67-f053-4393-99b1-71151c466434 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f3695e3-ec59-4fef-9596-926ec623c669 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5f3695e3-ec59-4fef-9596-926ec623c669 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6ac9b874-fb05-4d4b-806e-e7d9f8496026 \\u003e DN.FE.ExecuteQuery/661413b7-6319-4d4e-84ca-2ab054915379)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[20:54:03.095]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:53:59.8371044Z);\\nlet _endTime = datetime(2026-08-24T20:53:59.8371051Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bzlbgl4sq6l2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:54:03.0699661Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;535f349f-0f02-4848-ad54-a4b9b368c05a\\\",\\r\\n \\\"activityId\\\": \\\"b8e19f30-0aeb-4bae-aa9a-2994c97ca8e2\\\",\\r\\n \\\"subActivityId\\\": \\\"c7079a92-9d3f-43af-ae10-33c5d509b1db\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"fe309183-9dd8-40b4-8fca-7e5ee982bcdd\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;535f349f-0f02-4848-ad54-a4b9b368c05a ARID=b8e19f30-0aeb-4bae-aa9a-2994c97ca8e2 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/30e8047d-b899-4df8-96bf-7f67a7ea3a80 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/30e8047d-b899-4df8-96bf-7f67a7ea3a80 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/fe309183-9dd8-40b4-8fca-7e5ee982bcdd \\u003e DN.FE.ExecuteQuery/c7079a92-9d3f-43af-ae10-33c5d509b1db)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:54:03.206]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:53:59.8371044Z);\\nlet _endTime = datetime(2026-08-24T20:53:59.8371051Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bzlbgl4sq6l2'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:54:03.0699661Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;535f349f-0f02-4848-ad54-a4b9b368c05a\\\",\\r\\n \\\"activityId\\\": \\\"b8e19f30-0aeb-4bae-aa9a-2994c97ca8e2\\\",\\r\\n \\\"subActivityId\\\": \\\"c7079a92-9d3f-43af-ae10-33c5d509b1db\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"fe309183-9dd8-40b4-8fca-7e5ee982bcdd\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;535f349f-0f02-4848-ad54-a4b9b368c05a ARID=b8e19f30-0aeb-4bae-aa9a-2994c97ca8e2 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/30e8047d-b899-4df8-96bf-7f67a7ea3a80 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/30e8047d-b899-4df8-96bf-7f67a7ea3a80 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/fe309183-9dd8-40b4-8fca-7e5ee982bcdd \\u003e DN.FE.ExecuteQuery/c7079a92-9d3f-43af-ae10-33c5d509b1db)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977503907254272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6658,"pr_state":"open","pr_sha":"9906d38df8dc349a74b8507e4059d71747a87617","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T19:54:38Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"02ae1f9497b982cdf5f3e4b5b2b0b7aab4030112422fc979a4285980317f2ab6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977503907254272","occurred_at":"2026-08-24T19:54:38Z","failed_at":"e2e","signature_id":"eb508bccb119e029161a871dee87adcf17a97ac2d0acf65b89eac6aede0f896a","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-20 with version 4.20.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00031f900\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tczrqg\" in resourcegroup=\"rg-np-version-upgrade-tczrqg-fhlnbg67qjlz\" to finish creating: GET https://rp.j7254272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d7b313-a7f0-40de-93b0-918f61dd2377\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d7b313-a7f0-40de-93b0-918f61dd2377\",\n \"name\": \"c2d7b313-a7f0-40de-93b0-918f61dd2377\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T20:58:35.24134299Z\",\n \"endTime\": \"2026-08-24T21:17:39.53487523Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-20 with version 4.20.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00031f900\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tczrqg\" in resourcegroup=\"rg-np-version-upgrade-tczrqg-fhlnbg67qjlz\" to finish creating: GET https://rp.j7254272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d7b313-a7f0-40de-93b0-918f61dd2377\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d7b313-a7f0-40de-93b0-918f61dd2377\",\n \"name\": \"c2d7b313-a7f0-40de-93b0-918f61dd2377\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T20:58:35.24134299Z\",\n \"endTime\": \"2026-08-24T21:17:39.53487523Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a622071b3b2fdd70b9dfd28c28850ed4c1148c430840b906bc7ea92ea6b68665","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977503907254272","occurred_at":"2026-08-24T19:54:38Z","failed_at":"e2e","signature_id":"0b9a16666408f3f71c77e5a20e559de4e428d06efaede29056c5fdec679a0549","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 603.830s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0004931d0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:15:16.025]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:15:11.3931787Z);\\nlet _endTime = datetime(2026-08-24T21:15:11.3931797Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8xqfrdjbw4lb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:15:16.0014048Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f906d12a-14d5-4726-b29d-abd04f327f3a\\\",\\r\\n \\\"activityId\\\": \\\"2e49c6f6-575c-42f6-b8f3-dad7f49c8a1e\\\",\\r\\n \\\"subActivityId\\\": \\\"aa8f470b-0b53-475d-a4fa-cf95cf2df525\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c446787f-4208-43bb-af92-7f3c5bab6df5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f906d12a-14d5-4726-b29d-abd04f327f3a ARID=2e49c6f6-575c-42f6-b8f3-dad7f49c8a1e \\u003e GW.Http.CallContext/2e49c6f6-575c-42f6-b8f3-dad7f49c8a1e \\u003e GW.ExecuteQuery/c446787f-4208-43bb-af92-7f3c5bab6df5 \\u003e DN.FE.ExecuteQuery/aa8f470b-0b53-475d-a4fa-cf95cf2df525)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:15:16.083]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:15:11.3931787Z);\\nlet _endTime = datetime(2026-08-24T21:15:11.3931797Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8xqfrdjbw4lb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:15:16.0014048Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f906d12a-14d5-4726-b29d-abd04f327f3a\\\",\\r\\n \\\"activityId\\\": \\\"2e49c6f6-575c-42f6-b8f3-dad7f49c8a1e\\\",\\r\\n \\\"subActivityId\\\": \\\"aa8f470b-0b53-475d-a4fa-cf95cf2df525\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c446787f-4208-43bb-af92-7f3c5bab6df5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f906d12a-14d5-4726-b29d-abd04f327f3a ARID=2e49c6f6-575c-42f6-b8f3-dad7f49c8a1e \\u003e GW.Http.CallContext/2e49c6f6-575c-42f6-b8f3-dad7f49c8a1e \\u003e GW.ExecuteQuery/c446787f-4208-43bb-af92-7f3c5bab6df5 \\u003e DN.FE.ExecuteQuery/aa8f470b-0b53-475d-a4fa-cf95cf2df525)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:15:19.746]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:15:16.2075639Z);\\nlet _endTime = datetime(2026-08-24T21:15:16.2075645Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8xqfrdjbw4lb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:15:19.7247115Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;52787429-40fb-41f6-897a-171e6f45b558\\\",\\r\\n \\\"activityId\\\": \\\"451ccf71-58ba-4e87-941d-9a32337dc0a7\\\",\\r\\n \\\"subActivityId\\\": \\\"fe044149-3b5c-44f8-a307-080742dbb5ff\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f8c25e56-5afe-4b10-bd5d-0dd97d7b4890\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;52787429-40fb-41f6-897a-171e6f45b558 ARID=451ccf71-58ba-4e87-941d-9a32337dc0a7 \\u003e GW.Http.CallContext/451ccf71-58ba-4e87-941d-9a32337dc0a7 \\u003e GW.ExecuteQuery/f8c25e56-5afe-4b10-bd5d-0dd97d7b4890 \\u003e DN.FE.ExecuteQuery/fe044149-3b5c-44f8-a307-080742dbb5ff)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:15:19.746]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"context canceled\",\n \"name\": \"cosmosResourceSnapshots\"\n }�[0m\n �[37m[21:15:19.814]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:15:16.2075639Z);\\nlet _endTime = datetime(2026-08-24T21:15:16.2075645Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8xqfrdjbw4lb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:15:19.7247115Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;52787429-40fb-41f6-897a-171e6f45b558\\\",\\r\\n \\\"activityId\\\": \\\"451ccf71-58ba-4e87-941d-9a32337dc0a7\\\",\\r\\n \\\"subActivityId\\\": \\\"fe044149-3b5c-44f8-a307-080742dbb5ff\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f8c25e56-5afe-4b10-bd5d-0dd97d7b4890\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;52787429-40fb-41f6-897a-171e6f45b558 ARID=451ccf71-58ba-4e87-941d-9a32337dc0a7 \\u003e GW.Http.CallContext/451ccf71-58ba-4e87-941d-9a32337dc0a7 \\u003e GW.ExecuteQuery/f8c25e56-5afe-4b10-bd5d-0dd97d7b4890 \\u003e DN.FE.ExecuteQuery/fe044149-3b5c-44f8-a307-080742dbb5ff)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:15:27.083]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:15:23.4959268Z);\\nlet _endTime = datetime(2026-08-24T21:15:23.4959274Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8xqfrdjbw4lb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:15:27.0563191Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a50f7a18-f495-4b29-82b8-dc128132d94e\\\",\\r\\n \\\"activityId\\\": \\\"c2e33394-53e9-4db1-b207-3f75282af04f\\\",\\r\\n \\\"subActivityId\\\": \\\"869609d0-5465-4389-80a4-03eb382e7d30\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b0dc1001-d29d-4093-9f9a-e8d6af8ae3e9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a50f7a18-f495-4b29-82b8-dc128132d94e ARID=c2e33394-53e9-4db1-b207-3f75282af04f \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/408a6271-9bb4-47a9-92f1-c223c7195aca \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/408a6271-9bb4-47a9-92f1-c223c7195aca \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/b0dc1001-d29d-4093-9f9a-e8d6af8ae3e9 \\u003e DN.FE.ExecuteQuery/869609d0-5465-4389-80a4-03eb382e7d30)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:15:27.198]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:15:23.4959268Z);\\nlet _endTime = datetime(2026-08-24T21:15:23.4959274Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8xqfrdjbw4lb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:15:27.0563191Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a50f7a18-f495-4b29-82b8-dc128132d94e\\\",\\r\\n \\\"activityId\\\": \\\"c2e33394-53e9-4db1-b207-3f75282af04f\\\",\\r\\n \\\"subActivityId\\\": \\\"869609d0-5465-4389-80a4-03eb382e7d30\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b0dc1001-d29d-4093-9f9a-e8d6af8ae3e9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a50f7a18-f495-4b29-82b8-dc128132d94e ARID=c2e33394-53e9-4db1-b207-3f75282af04f \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/408a6271-9bb4-47a9-92f1-c223c7195aca \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/408a6271-9bb4-47a9-92f1-c223c7195aca \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/b0dc1001-d29d-4093-9f9a-e8d6af8ae3e9 \\u003e DN.FE.ExecuteQuery/869609d0-5465-4389-80a4-03eb382e7d30)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 603.830s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0004931d0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:15:16.025]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:15:11.3931787Z);\\nlet _endTime = datetime(2026-08-24T21:15:11.3931797Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8xqfrdjbw4lb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:15:16.0014048Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f906d12a-14d5-4726-b29d-abd04f327f3a\\\",\\r\\n \\\"activityId\\\": \\\"2e49c6f6-575c-42f6-b8f3-dad7f49c8a1e\\\",\\r\\n \\\"subActivityId\\\": \\\"aa8f470b-0b53-475d-a4fa-cf95cf2df525\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c446787f-4208-43bb-af92-7f3c5bab6df5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f906d12a-14d5-4726-b29d-abd04f327f3a ARID=2e49c6f6-575c-42f6-b8f3-dad7f49c8a1e \\u003e GW.Http.CallContext/2e49c6f6-575c-42f6-b8f3-dad7f49c8a1e \\u003e GW.ExecuteQuery/c446787f-4208-43bb-af92-7f3c5bab6df5 \\u003e DN.FE.ExecuteQuery/aa8f470b-0b53-475d-a4fa-cf95cf2df525)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:15:16.083]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:15:11.3931787Z);\\nlet _endTime = datetime(2026-08-24T21:15:11.3931797Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8xqfrdjbw4lb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:15:16.0014048Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f906d12a-14d5-4726-b29d-abd04f327f3a\\\",\\r\\n \\\"activityId\\\": \\\"2e49c6f6-575c-42f6-b8f3-dad7f49c8a1e\\\",\\r\\n \\\"subActivityId\\\": \\\"aa8f470b-0b53-475d-a4fa-cf95cf2df525\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c446787f-4208-43bb-af92-7f3c5bab6df5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f906d12a-14d5-4726-b29d-abd04f327f3a ARID=2e49c6f6-575c-42f6-b8f3-dad7f49c8a1e \\u003e GW.Http.CallContext/2e49c6f6-575c-42f6-b8f3-dad7f49c8a1e \\u003e GW.ExecuteQuery/c446787f-4208-43bb-af92-7f3c5bab6df5 \\u003e DN.FE.ExecuteQuery/aa8f470b-0b53-475d-a4fa-cf95cf2df525)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:15:19.746]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:15:16.2075639Z);\\nlet _endTime = datetime(2026-08-24T21:15:16.2075645Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8xqfrdjbw4lb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:15:19.7247115Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;52787429-40fb-41f6-897a-171e6f45b558\\\",\\r\\n \\\"activityId\\\": \\\"451ccf71-58ba-4e87-941d-9a32337dc0a7\\\",\\r\\n \\\"subActivityId\\\": \\\"fe044149-3b5c-44f8-a307-080742dbb5ff\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f8c25e56-5afe-4b10-bd5d-0dd97d7b4890\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;52787429-40fb-41f6-897a-171e6f45b558 ARID=451ccf71-58ba-4e87-941d-9a32337dc0a7 \\u003e GW.Http.CallContext/451ccf71-58ba-4e87-941d-9a32337dc0a7 \\u003e GW.ExecuteQuery/f8c25e56-5afe-4b10-bd5d-0dd97d7b4890 \\u003e DN.FE.ExecuteQuery/fe044149-3b5c-44f8-a307-080742dbb5ff)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:15:19.746]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"context canceled\",\n \"name\": \"cosmosResourceSnapshots\"\n }�[0m\n �[37m[21:15:19.814]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:15:16.2075639Z);\\nlet _endTime = datetime(2026-08-24T21:15:16.2075645Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8xqfrdjbw4lb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:15:19.7247115Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;52787429-40fb-41f6-897a-171e6f45b558\\\",\\r\\n \\\"activityId\\\": \\\"451ccf71-58ba-4e87-941d-9a32337dc0a7\\\",\\r\\n \\\"subActivityId\\\": \\\"fe044149-3b5c-44f8-a307-080742dbb5ff\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f8c25e56-5afe-4b10-bd5d-0dd97d7b4890\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;52787429-40fb-41f6-897a-171e6f45b558 ARID=451ccf71-58ba-4e87-941d-9a32337dc0a7 \\u003e GW.Http.CallContext/451ccf71-58ba-4e87-941d-9a32337dc0a7 \\u003e GW.ExecuteQuery/f8c25e56-5afe-4b10-bd5d-0dd97d7b4890 \\u003e DN.FE.ExecuteQuery/fe044149-3b5c-44f8-a307-080742dbb5ff)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:15:27.083]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:15:23.4959268Z);\\nlet _endTime = datetime(2026-08-24T21:15:23.4959274Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8xqfrdjbw4lb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:15:27.0563191Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a50f7a18-f495-4b29-82b8-dc128132d94e\\\",\\r\\n \\\"activityId\\\": \\\"c2e33394-53e9-4db1-b207-3f75282af04f\\\",\\r\\n \\\"subActivityId\\\": \\\"869609d0-5465-4389-80a4-03eb382e7d30\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b0dc1001-d29d-4093-9f9a-e8d6af8ae3e9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a50f7a18-f495-4b29-82b8-dc128132d94e ARID=c2e33394-53e9-4db1-b207-3f75282af04f \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/408a6271-9bb4-47a9-92f1-c223c7195aca \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/408a6271-9bb4-47a9-92f1-c223c7195aca \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/b0dc1001-d29d-4093-9f9a-e8d6af8ae3e9 \\u003e DN.FE.ExecuteQuery/869609d0-5465-4389-80a4-03eb382e7d30)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:15:27.198]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:15:23.4959268Z);\\nlet _endTime = datetime(2026-08-24T21:15:23.4959274Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-8xqfrdjbw4lb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:15:27.0563191Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a50f7a18-f495-4b29-82b8-dc128132d94e\\\",\\r\\n \\\"activityId\\\": \\\"c2e33394-53e9-4db1-b207-3f75282af04f\\\",\\r\\n \\\"subActivityId\\\": \\\"869609d0-5465-4389-80a4-03eb382e7d30\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b0dc1001-d29d-4093-9f9a-e8d6af8ae3e9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a50f7a18-f495-4b29-82b8-dc128132d94e ARID=c2e33394-53e9-4db1-b207-3f75282af04f \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/408a6271-9bb4-47a9-92f1-c223c7195aca \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/408a6271-9bb4-47a9-92f1-c223c7195aca \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/b0dc1001-d29d-4093-9f9a-e8d6af8ae3e9 \\u003e DN.FE.ExecuteQuery/869609d0-5465-4389-80a4-03eb382e7d30)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977495384428544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6629,"pr_state":"open","pr_sha":"c6d46781a7abcc4c551cdf9edbb601d3aee0cc38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T19:54:36Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"58a3bbf5623e4fdfce69e40218060645fcde7b75b657d69451c1360f3fcf6e64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977495384428544","occurred_at":"2026-08-24T19:54:36Z","failed_at":"e2e","signature_id":"07fcf787e9667bb548a1ae4e2aa892f82683b85aa913615ccc3c55f328e44f13","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.008s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0018343f0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:11:48.168]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:11:45.1785302Z);\\nlet _endTime = datetime(2026-08-24T21:11:45.1785306Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nhrxvm5llfcn'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:11:48.1484338Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ae0286df-584b-4a34-9f74-dcee83455e56\\\",\\r\\n \\\"activityId\\\": \\\"5bf3a5e5-fd68-4ea3-9729-39d43d93c8ec\\\",\\r\\n \\\"subActivityId\\\": \\\"dfac13ec-813c-4196-8cc2-2e9a57e63812\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"101755fc-9d7c-424e-906e-dab6c556873c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ae0286df-584b-4a34-9f74-dcee83455e56 ARID=5bf3a5e5-fd68-4ea3-9729-39d43d93c8ec \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b44f371f-d679-44e7-8b17-97d091eb055d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b44f371f-d679-44e7-8b17-97d091eb055d \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/101755fc-9d7c-424e-906e-dab6c556873c \\u003e DN.FE.ExecuteQuery/dfac13ec-813c-4196-8cc2-2e9a57e63812)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:11:48.227]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:11:45.1785302Z);\\nlet _endTime = datetime(2026-08-24T21:11:45.1785306Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nhrxvm5llfcn'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:11:48.1484338Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ae0286df-584b-4a34-9f74-dcee83455e56\\\",\\r\\n \\\"activityId\\\": \\\"5bf3a5e5-fd68-4ea3-9729-39d43d93c8ec\\\",\\r\\n \\\"subActivityId\\\": \\\"dfac13ec-813c-4196-8cc2-2e9a57e63812\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"101755fc-9d7c-424e-906e-dab6c556873c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ae0286df-584b-4a34-9f74-dcee83455e56 ARID=5bf3a5e5-fd68-4ea3-9729-39d43d93c8ec \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b44f371f-d679-44e7-8b17-97d091eb055d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b44f371f-d679-44e7-8b17-97d091eb055d \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/101755fc-9d7c-424e-906e-dab6c556873c \\u003e DN.FE.ExecuteQuery/dfac13ec-813c-4196-8cc2-2e9a57e63812)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:11:51.205]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:11:48.3065804Z);\\nlet _endTime = datetime(2026-08-24T21:11:48.3065811Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nhrxvm5llfcn'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:11:51.1836113Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ee12add3-3beb-40dc-9f72-af09feb40f1e\\\",\\r\\n \\\"activityId\\\": \\\"aabba74b-59ad-44c8-9241-3e6c721648d8\\\",\\r\\n \\\"subActivityId\\\": \\\"4b05f656-fa05-436e-8ee6-fa3cad2e5460\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2c8d9f77-bbab-4efe-829b-2617c0d7469a\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ee12add3-3beb-40dc-9f72-af09feb40f1e ARID=aabba74b-59ad-44c8-9241-3e6c721648d8 \\u003e GW.Http.CallContext/aabba74b-59ad-44c8-9241-3e6c721648d8 \\u003e GW.ExecuteQuery/2c8d9f77-bbab-4efe-829b-2617c0d7469a \\u003e DN.FE.ExecuteQuery/4b05f656-fa05-436e-8ee6-fa3cad2e5460)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:11:51.256]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:11:48.3065804Z);\\nlet _endTime = datetime(2026-08-24T21:11:48.3065811Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nhrxvm5llfcn'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:11:51.1836113Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ee12add3-3beb-40dc-9f72-af09feb40f1e\\\",\\r\\n \\\"activityId\\\": \\\"aabba74b-59ad-44c8-9241-3e6c721648d8\\\",\\r\\n \\\"subActivityId\\\": \\\"4b05f656-fa05-436e-8ee6-fa3cad2e5460\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2c8d9f77-bbab-4efe-829b-2617c0d7469a\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ee12add3-3beb-40dc-9f72-af09feb40f1e ARID=aabba74b-59ad-44c8-9241-3e6c721648d8 \\u003e GW.Http.CallContext/aabba74b-59ad-44c8-9241-3e6c721648d8 \\u003e GW.ExecuteQuery/2c8d9f77-bbab-4efe-829b-2617c0d7469a \\u003e DN.FE.ExecuteQuery/4b05f656-fa05-436e-8ee6-fa3cad2e5460)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:11:57.885]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:11:54.6424073Z);\\nlet _endTime = datetime(2026-08-24T21:11:54.6424077Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nhrxvm5llfcn'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:11:57.8650613Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c2c9a2ce-5b50-4831-9308-2de17897e7e8\\\",\\r\\n \\\"activityId\\\": \\\"61421d75-0512-490c-ae8e-ed449f7b4ef1\\\",\\r\\n \\\"subActivityId\\\": \\\"cddb715f-ae58-4ad3-8057-d1e4ab4131aa\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"af3adfb7-44fa-4141-826e-04cfbf96ab82\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c2c9a2ce-5b50-4831-9308-2de17897e7e8 ARID=61421d75-0512-490c-ae8e-ed449f7b4ef1 \\u003e GW.Http.CallContext/61421d75-0512-490c-ae8e-ed449f7b4ef1 \\u003e GW.ExecuteQuery/af3adfb7-44fa-4141-826e-04cfbf96ab82 \\u003e DN.FE.ExecuteQuery/cddb715f-ae58-4ad3-8057-d1e4ab4131aa)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:11:57.986]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:11:54.6424073Z);\\nlet _endTime = datetime(2026-08-24T21:11:54.6424077Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nhrxvm5llfcn'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:11:57.8650613Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c2c9a2ce-5b50-4831-9308-2de17897e7e8\\\",\\r\\n \\\"activityId\\\": \\\"61421d75-0512-490c-ae8e-ed449f7b4ef1\\\",\\r\\n \\\"subActivityId\\\": \\\"cddb715f-ae58-4ad3-8057-d1e4ab4131aa\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"af3adfb7-44fa-4141-826e-04cfbf96ab82\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c2c9a2ce-5b50-4831-9308-2de17897e7e8 ARID=61421d75-0512-490c-ae8e-ed449f7b4ef1 \\u003e GW.Http.CallContext/61421d75-0512-490c-ae8e-ed449f7b4ef1 \\u003e GW.ExecuteQuery/af3adfb7-44fa-4141-826e-04cfbf96ab82 \\u003e DN.FE.ExecuteQuery/cddb715f-ae58-4ad3-8057-d1e4ab4131aa)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.008s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0018343f0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:11:48.168]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:11:45.1785302Z);\\nlet _endTime = datetime(2026-08-24T21:11:45.1785306Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nhrxvm5llfcn'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:11:48.1484338Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ae0286df-584b-4a34-9f74-dcee83455e56\\\",\\r\\n \\\"activityId\\\": \\\"5bf3a5e5-fd68-4ea3-9729-39d43d93c8ec\\\",\\r\\n \\\"subActivityId\\\": \\\"dfac13ec-813c-4196-8cc2-2e9a57e63812\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"101755fc-9d7c-424e-906e-dab6c556873c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ae0286df-584b-4a34-9f74-dcee83455e56 ARID=5bf3a5e5-fd68-4ea3-9729-39d43d93c8ec \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b44f371f-d679-44e7-8b17-97d091eb055d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b44f371f-d679-44e7-8b17-97d091eb055d \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/101755fc-9d7c-424e-906e-dab6c556873c \\u003e DN.FE.ExecuteQuery/dfac13ec-813c-4196-8cc2-2e9a57e63812)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:11:48.227]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:11:45.1785302Z);\\nlet _endTime = datetime(2026-08-24T21:11:45.1785306Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nhrxvm5llfcn'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:11:48.1484338Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ae0286df-584b-4a34-9f74-dcee83455e56\\\",\\r\\n \\\"activityId\\\": \\\"5bf3a5e5-fd68-4ea3-9729-39d43d93c8ec\\\",\\r\\n \\\"subActivityId\\\": \\\"dfac13ec-813c-4196-8cc2-2e9a57e63812\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"101755fc-9d7c-424e-906e-dab6c556873c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ae0286df-584b-4a34-9f74-dcee83455e56 ARID=5bf3a5e5-fd68-4ea3-9729-39d43d93c8ec \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b44f371f-d679-44e7-8b17-97d091eb055d \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b44f371f-d679-44e7-8b17-97d091eb055d \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/101755fc-9d7c-424e-906e-dab6c556873c \\u003e DN.FE.ExecuteQuery/dfac13ec-813c-4196-8cc2-2e9a57e63812)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:11:51.205]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:11:48.3065804Z);\\nlet _endTime = datetime(2026-08-24T21:11:48.3065811Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nhrxvm5llfcn'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:11:51.1836113Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ee12add3-3beb-40dc-9f72-af09feb40f1e\\\",\\r\\n \\\"activityId\\\": \\\"aabba74b-59ad-44c8-9241-3e6c721648d8\\\",\\r\\n \\\"subActivityId\\\": \\\"4b05f656-fa05-436e-8ee6-fa3cad2e5460\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2c8d9f77-bbab-4efe-829b-2617c0d7469a\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ee12add3-3beb-40dc-9f72-af09feb40f1e ARID=aabba74b-59ad-44c8-9241-3e6c721648d8 \\u003e GW.Http.CallContext/aabba74b-59ad-44c8-9241-3e6c721648d8 \\u003e GW.ExecuteQuery/2c8d9f77-bbab-4efe-829b-2617c0d7469a \\u003e DN.FE.ExecuteQuery/4b05f656-fa05-436e-8ee6-fa3cad2e5460)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:11:51.256]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:11:48.3065804Z);\\nlet _endTime = datetime(2026-08-24T21:11:48.3065811Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nhrxvm5llfcn'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:11:51.1836113Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;ee12add3-3beb-40dc-9f72-af09feb40f1e\\\",\\r\\n \\\"activityId\\\": \\\"aabba74b-59ad-44c8-9241-3e6c721648d8\\\",\\r\\n \\\"subActivityId\\\": \\\"4b05f656-fa05-436e-8ee6-fa3cad2e5460\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2c8d9f77-bbab-4efe-829b-2617c0d7469a\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;ee12add3-3beb-40dc-9f72-af09feb40f1e ARID=aabba74b-59ad-44c8-9241-3e6c721648d8 \\u003e GW.Http.CallContext/aabba74b-59ad-44c8-9241-3e6c721648d8 \\u003e GW.ExecuteQuery/2c8d9f77-bbab-4efe-829b-2617c0d7469a \\u003e DN.FE.ExecuteQuery/4b05f656-fa05-436e-8ee6-fa3cad2e5460)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:11:57.885]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:11:54.6424073Z);\\nlet _endTime = datetime(2026-08-24T21:11:54.6424077Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nhrxvm5llfcn'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:11:57.8650613Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c2c9a2ce-5b50-4831-9308-2de17897e7e8\\\",\\r\\n \\\"activityId\\\": \\\"61421d75-0512-490c-ae8e-ed449f7b4ef1\\\",\\r\\n \\\"subActivityId\\\": \\\"cddb715f-ae58-4ad3-8057-d1e4ab4131aa\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"af3adfb7-44fa-4141-826e-04cfbf96ab82\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c2c9a2ce-5b50-4831-9308-2de17897e7e8 ARID=61421d75-0512-490c-ae8e-ed449f7b4ef1 \\u003e GW.Http.CallContext/61421d75-0512-490c-ae8e-ed449f7b4ef1 \\u003e GW.ExecuteQuery/af3adfb7-44fa-4141-826e-04cfbf96ab82 \\u003e DN.FE.ExecuteQuery/cddb715f-ae58-4ad3-8057-d1e4ab4131aa)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:11:57.986]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:11:54.6424073Z);\\nlet _endTime = datetime(2026-08-24T21:11:54.6424077Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nhrxvm5llfcn'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:11:57.8650613Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c2c9a2ce-5b50-4831-9308-2de17897e7e8\\\",\\r\\n \\\"activityId\\\": \\\"61421d75-0512-490c-ae8e-ed449f7b4ef1\\\",\\r\\n \\\"subActivityId\\\": \\\"cddb715f-ae58-4ad3-8057-d1e4ab4131aa\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"af3adfb7-44fa-4141-826e-04cfbf96ab82\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c2c9a2ce-5b50-4831-9308-2de17897e7e8 ARID=61421d75-0512-490c-ae8e-ed449f7b4ef1 \\u003e GW.Http.CallContext/61421d75-0512-490c-ae8e-ed449f7b4ef1 \\u003e GW.ExecuteQuery/af3adfb7-44fa-4141-826e-04cfbf96ab82 \\u003e DN.FE.ExecuteQuery/cddb715f-ae58-4ad3-8057-d1e4ab4131aa)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"9d8297a5d1b938eee1863a3fcacf8231149eed1e07373bdb55dab04b38b0c610","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977495384428544","occurred_at":"2026-08-24T19:54:36Z","failed_at":"e2e","signature_id":"4a86fd68539d0c65cc2c794f0a3b117d62969a34ae1b535d87cc0cce6d88b528","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-fn4s2m\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dea2e0\u003e: \n DNS for route host agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-fn4s2m\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dea2e0\u003e: \n DNS for route host agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977485536202752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5958,"pr_state":"open","pr_sha":"8ce3ed6e8d91907caf45e2b3ec8c94ff7141fe0a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T19:54:34Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bf35fecc4cf6cc9b3f263c27d597355eefcabea270e3e1aa8af1af613f00cab4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977485536202752","occurred_at":"2026-08-24T19:54:34Z","failed_at":"e2e","signature_id":"edf4f7b0d5436e65624e47252a3c9819a947e53aeb06b94097e54f965d358ffa","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.008s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b18230\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:06:58.789]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:06:55.4059232Z);\\nlet _endTime = datetime(2026-08-24T21:06:55.4059238Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-qdhqs95kdb4g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:06:58.7667615Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;38bc470d-9366-4b94-8899-99c4c881f92e\\\",\\r\\n \\\"activityId\\\": \\\"809b48ce-4db2-451c-81c2-e41a8748a2cb\\\",\\r\\n \\\"subActivityId\\\": \\\"2b597bb5-a965-4295-9801-672a05444170\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9730504b-885f-48fc-870b-b1d8ffdc8543\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;38bc470d-9366-4b94-8899-99c4c881f92e ARID=809b48ce-4db2-451c-81c2-e41a8748a2cb \\u003e GW.Http.CallContext/809b48ce-4db2-451c-81c2-e41a8748a2cb \\u003e GW.ExecuteQuery/9730504b-885f-48fc-870b-b1d8ffdc8543 \\u003e DN.FE.ExecuteQuery/2b597bb5-a965-4295-9801-672a05444170)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:06:58.843]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:06:55.4059232Z);\\nlet _endTime = datetime(2026-08-24T21:06:55.4059238Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-qdhqs95kdb4g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:06:58.7667615Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;38bc470d-9366-4b94-8899-99c4c881f92e\\\",\\r\\n \\\"activityId\\\": \\\"809b48ce-4db2-451c-81c2-e41a8748a2cb\\\",\\r\\n \\\"subActivityId\\\": \\\"2b597bb5-a965-4295-9801-672a05444170\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9730504b-885f-48fc-870b-b1d8ffdc8543\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;38bc470d-9366-4b94-8899-99c4c881f92e ARID=809b48ce-4db2-451c-81c2-e41a8748a2cb \\u003e GW.Http.CallContext/809b48ce-4db2-451c-81c2-e41a8748a2cb \\u003e GW.ExecuteQuery/9730504b-885f-48fc-870b-b1d8ffdc8543 \\u003e DN.FE.ExecuteQuery/2b597bb5-a965-4295-9801-672a05444170)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:07:02.136]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:06:58.9135625Z);\\nlet _endTime = datetime(2026-08-24T21:06:58.9135630Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-qdhqs95kdb4g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:07:02.1107130Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;883d4f16-b2d5-473d-ad61-2e2eaafa2ac2\\\",\\r\\n \\\"activityId\\\": \\\"14258bf4-50f1-41c9-b908-681c17389fa4\\\",\\r\\n \\\"subActivityId\\\": \\\"da07376e-e1ee-4ad8-95bf-f24b5bb180b7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"47829b11-3403-462c-83ee-05b021d04b47\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;883d4f16-b2d5-473d-ad61-2e2eaafa2ac2 ARID=14258bf4-50f1-41c9-b908-681c17389fa4 \\u003e GW.Http.CallContext/14258bf4-50f1-41c9-b908-681c17389fa4 \\u003e GW.ExecuteQuery/47829b11-3403-462c-83ee-05b021d04b47 \\u003e DN.FE.ExecuteQuery/da07376e-e1ee-4ad8-95bf-f24b5bb180b7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:07:02.183]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:06:58.9135625Z);\\nlet _endTime = datetime(2026-08-24T21:06:58.9135630Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-qdhqs95kdb4g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:07:02.1107130Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;883d4f16-b2d5-473d-ad61-2e2eaafa2ac2\\\",\\r\\n \\\"activityId\\\": \\\"14258bf4-50f1-41c9-b908-681c17389fa4\\\",\\r\\n \\\"subActivityId\\\": \\\"da07376e-e1ee-4ad8-95bf-f24b5bb180b7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"47829b11-3403-462c-83ee-05b021d04b47\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;883d4f16-b2d5-473d-ad61-2e2eaafa2ac2 ARID=14258bf4-50f1-41c9-b908-681c17389fa4 \\u003e GW.Http.CallContext/14258bf4-50f1-41c9-b908-681c17389fa4 \\u003e GW.ExecuteQuery/47829b11-3403-462c-83ee-05b021d04b47 \\u003e DN.FE.ExecuteQuery/da07376e-e1ee-4ad8-95bf-f24b5bb180b7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:07:10.268]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:07:06.0371001Z);\\nlet _endTime = datetime(2026-08-24T21:07:06.0371007Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-qdhqs95kdb4g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:07:10.2479420Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d4eb7db5-1698-4436-b648-0abc283404c8\\\",\\r\\n \\\"activityId\\\": \\\"428c9294-ade7-4000-8c97-cf84299bfe87\\\",\\r\\n \\\"subActivityId\\\": \\\"845f816d-2c80-4bdc-8b63-dfb3d52faa8d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"eaf67d4e-1868-4364-b7bb-9b967c69c037\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d4eb7db5-1698-4436-b648-0abc283404c8 ARID=428c9294-ade7-4000-8c97-cf84299bfe87 \\u003e GW.Http.CallContext/428c9294-ade7-4000-8c97-cf84299bfe87 \\u003e GW.ExecuteQuery/eaf67d4e-1868-4364-b7bb-9b967c69c037 \\u003e DN.FE.ExecuteQuery/845f816d-2c80-4bdc-8b63-dfb3d52faa8d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:07:10.361]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:07:06.0371001Z);\\nlet _endTime = datetime(2026-08-24T21:07:06.0371007Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-qdhqs95kdb4g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:07:10.2479420Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d4eb7db5-1698-4436-b648-0abc283404c8\\\",\\r\\n \\\"activityId\\\": \\\"428c9294-ade7-4000-8c97-cf84299bfe87\\\",\\r\\n \\\"subActivityId\\\": \\\"845f816d-2c80-4bdc-8b63-dfb3d52faa8d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"eaf67d4e-1868-4364-b7bb-9b967c69c037\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d4eb7db5-1698-4436-b648-0abc283404c8 ARID=428c9294-ade7-4000-8c97-cf84299bfe87 \\u003e GW.Http.CallContext/428c9294-ade7-4000-8c97-cf84299bfe87 \\u003e GW.ExecuteQuery/eaf67d4e-1868-4364-b7bb-9b967c69c037 \\u003e DN.FE.ExecuteQuery/845f816d-2c80-4bdc-8b63-dfb3d52faa8d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.008s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b18230\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[21:06:58.789]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:06:55.4059232Z);\\nlet _endTime = datetime(2026-08-24T21:06:55.4059238Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-qdhqs95kdb4g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:06:58.7667615Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;38bc470d-9366-4b94-8899-99c4c881f92e\\\",\\r\\n \\\"activityId\\\": \\\"809b48ce-4db2-451c-81c2-e41a8748a2cb\\\",\\r\\n \\\"subActivityId\\\": \\\"2b597bb5-a965-4295-9801-672a05444170\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9730504b-885f-48fc-870b-b1d8ffdc8543\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;38bc470d-9366-4b94-8899-99c4c881f92e ARID=809b48ce-4db2-451c-81c2-e41a8748a2cb \\u003e GW.Http.CallContext/809b48ce-4db2-451c-81c2-e41a8748a2cb \\u003e GW.ExecuteQuery/9730504b-885f-48fc-870b-b1d8ffdc8543 \\u003e DN.FE.ExecuteQuery/2b597bb5-a965-4295-9801-672a05444170)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:06:58.843]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:06:55.4059232Z);\\nlet _endTime = datetime(2026-08-24T21:06:55.4059238Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-qdhqs95kdb4g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:06:58.7667615Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;38bc470d-9366-4b94-8899-99c4c881f92e\\\",\\r\\n \\\"activityId\\\": \\\"809b48ce-4db2-451c-81c2-e41a8748a2cb\\\",\\r\\n \\\"subActivityId\\\": \\\"2b597bb5-a965-4295-9801-672a05444170\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"9730504b-885f-48fc-870b-b1d8ffdc8543\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;38bc470d-9366-4b94-8899-99c4c881f92e ARID=809b48ce-4db2-451c-81c2-e41a8748a2cb \\u003e GW.Http.CallContext/809b48ce-4db2-451c-81c2-e41a8748a2cb \\u003e GW.ExecuteQuery/9730504b-885f-48fc-870b-b1d8ffdc8543 \\u003e DN.FE.ExecuteQuery/2b597bb5-a965-4295-9801-672a05444170)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[21:07:02.136]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:06:58.9135625Z);\\nlet _endTime = datetime(2026-08-24T21:06:58.9135630Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-qdhqs95kdb4g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:07:02.1107130Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;883d4f16-b2d5-473d-ad61-2e2eaafa2ac2\\\",\\r\\n \\\"activityId\\\": \\\"14258bf4-50f1-41c9-b908-681c17389fa4\\\",\\r\\n \\\"subActivityId\\\": \\\"da07376e-e1ee-4ad8-95bf-f24b5bb180b7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"47829b11-3403-462c-83ee-05b021d04b47\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;883d4f16-b2d5-473d-ad61-2e2eaafa2ac2 ARID=14258bf4-50f1-41c9-b908-681c17389fa4 \\u003e GW.Http.CallContext/14258bf4-50f1-41c9-b908-681c17389fa4 \\u003e GW.ExecuteQuery/47829b11-3403-462c-83ee-05b021d04b47 \\u003e DN.FE.ExecuteQuery/da07376e-e1ee-4ad8-95bf-f24b5bb180b7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:07:02.183]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:06:58.9135625Z);\\nlet _endTime = datetime(2026-08-24T21:06:58.9135630Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-qdhqs95kdb4g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:07:02.1107130Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;883d4f16-b2d5-473d-ad61-2e2eaafa2ac2\\\",\\r\\n \\\"activityId\\\": \\\"14258bf4-50f1-41c9-b908-681c17389fa4\\\",\\r\\n \\\"subActivityId\\\": \\\"da07376e-e1ee-4ad8-95bf-f24b5bb180b7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"47829b11-3403-462c-83ee-05b021d04b47\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;883d4f16-b2d5-473d-ad61-2e2eaafa2ac2 ARID=14258bf4-50f1-41c9-b908-681c17389fa4 \\u003e GW.Http.CallContext/14258bf4-50f1-41c9-b908-681c17389fa4 \\u003e GW.ExecuteQuery/47829b11-3403-462c-83ee-05b021d04b47 \\u003e DN.FE.ExecuteQuery/da07376e-e1ee-4ad8-95bf-f24b5bb180b7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[21:07:10.268]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:07:06.0371001Z);\\nlet _endTime = datetime(2026-08-24T21:07:06.0371007Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-qdhqs95kdb4g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:07:10.2479420Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d4eb7db5-1698-4436-b648-0abc283404c8\\\",\\r\\n \\\"activityId\\\": \\\"428c9294-ade7-4000-8c97-cf84299bfe87\\\",\\r\\n \\\"subActivityId\\\": \\\"845f816d-2c80-4bdc-8b63-dfb3d52faa8d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"eaf67d4e-1868-4364-b7bb-9b967c69c037\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d4eb7db5-1698-4436-b648-0abc283404c8 ARID=428c9294-ade7-4000-8c97-cf84299bfe87 \\u003e GW.Http.CallContext/428c9294-ade7-4000-8c97-cf84299bfe87 \\u003e GW.ExecuteQuery/eaf67d4e-1868-4364-b7bb-9b967c69c037 \\u003e DN.FE.ExecuteQuery/845f816d-2c80-4bdc-8b63-dfb3d52faa8d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[21:07:10.361]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T21:07:06.0371001Z);\\nlet _endTime = datetime(2026-08-24T21:07:06.0371007Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-qdhqs95kdb4g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T21:07:10.2479420Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d4eb7db5-1698-4436-b648-0abc283404c8\\\",\\r\\n \\\"activityId\\\": \\\"428c9294-ade7-4000-8c97-cf84299bfe87\\\",\\r\\n \\\"subActivityId\\\": \\\"845f816d-2c80-4bdc-8b63-dfb3d52faa8d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"eaf67d4e-1868-4364-b7bb-9b967c69c037\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d4eb7db5-1698-4436-b648-0abc283404c8 ARID=428c9294-ade7-4000-8c97-cf84299bfe87 \\u003e GW.Http.CallContext/428c9294-ade7-4000-8c97-cf84299bfe87 \\u003e GW.ExecuteQuery/eaf67d4e-1868-4364-b7bb-9b967c69c037 \\u003e DN.FE.ExecuteQuery/845f816d-2c80-4bdc-8b63-dfb3d52faa8d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T19:06:19Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"86aebb3ace2e3061b9c9943d9bd7699bf23323349f4f1d75961b235edd94d20b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","failed_at":"e2e","signature_id":"7830c998ab91a0251740ef30429a36f78ee908a1781d172a1f172ebef622c56d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-q75lrtsn78tc/cluster-candidate-5-0-fph2pn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da0360\u003e: \n failed to create HCP cluster cluster-candidate-5-0-fph2pn: failed waiting for cluster=\"cluster-candidate-5-0-fph2pn\" in resourcegroup=\"rg-candidate-5-0-q75lrtsn78tc\" to finish creating: GET https://rp.j4792192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e34278a0-6b1d-4ae8-a229-75f5d648affe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e34278a0-6b1d-4ae8-a229-75f5d648affe\",\n \"name\": \"e34278a0-6b1d-4ae8-a229-75f5d648affe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T19:54:23.12480854Z\",\n \"endTime\": \"2026-08-24T20:13:28.144152741Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-q75lrtsn78tc/cluster-candidate-5-0-fph2pn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da0360\u003e: \n failed to create HCP cluster cluster-candidate-5-0-fph2pn: failed waiting for cluster=\"cluster-candidate-5-0-fph2pn\" in resourcegroup=\"rg-candidate-5-0-q75lrtsn78tc\" to finish creating: GET https://rp.j4792192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e34278a0-6b1d-4ae8-a229-75f5d648affe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e34278a0-6b1d-4ae8-a229-75f5d648affe\",\n \"name\": \"e34278a0-6b1d-4ae8-a229-75f5d648affe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T19:54:23.12480854Z\",\n \"endTime\": \"2026-08-24T20:13:28.144152741Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d1ba3f3b96eedf0b74dd35f2027cdfc8a6ef03be67272fe5d00da14568eff2bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","failed_at":"alert","signature_id":"d9b2b2f48eba30a813d1493c2cf9d779d74129e8e68690c29d48df3b77d5680a","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T20:07:35Z\n Ended: 2026-08-24T20:22:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4792192-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd56rvedr5tvpk5tgrh8ec0h63s5l3e-m5s0s1f2s8t9h9z\", pod=\"router-698586c75d-g7ftd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd56rvedr5tvpk5tgrh8ec0h63s5l3e-m5s0s1f2s8t9h9z/router-698586c75d-g7ftd has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"f186b43d37292c1ededfd7c77b0cd2e5af254e57ac2b0d71fd75bfef26b0d400","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","failed_at":"e2e","signature_id":"69234d1190be6bae030c9f0be83f507930b8aba09ce9643964f39fe12af78ce2","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00068eed0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[20:18:27.943]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:18:22.8770987Z);\\nlet _endTime = datetime(2026-08-24T20:18:22.8770991Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4gcjkhxx9dcw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:18:27.9010465Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;601b8990-928e-4bd5-baaf-10492f42f600\\\",\\r\\n \\\"activityId\\\": \\\"8c3b33c8-751b-4382-b6d8-5b5e1fb04797\\\",\\r\\n \\\"subActivityId\\\": \\\"1633d9fb-3c87-4c20-8b1a-e67b0bd3dac5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b1e0bb17-ad83-46fc-92fc-9d1509f8290e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;601b8990-928e-4bd5-baaf-10492f42f600 ARID=8c3b33c8-751b-4382-b6d8-5b5e1fb04797 \\u003e GW.Http.CallContext/8c3b33c8-751b-4382-b6d8-5b5e1fb04797 \\u003e GW.ExecuteQuery/b1e0bb17-ad83-46fc-92fc-9d1509f8290e \\u003e DN.FE.ExecuteQuery/1633d9fb-3c87-4c20-8b1a-e67b0bd3dac5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:18:28.002]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:18:22.8770987Z);\\nlet _endTime = datetime(2026-08-24T20:18:22.8770991Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4gcjkhxx9dcw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:18:27.9010465Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;601b8990-928e-4bd5-baaf-10492f42f600\\\",\\r\\n \\\"activityId\\\": \\\"8c3b33c8-751b-4382-b6d8-5b5e1fb04797\\\",\\r\\n \\\"subActivityId\\\": \\\"1633d9fb-3c87-4c20-8b1a-e67b0bd3dac5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b1e0bb17-ad83-46fc-92fc-9d1509f8290e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;601b8990-928e-4bd5-baaf-10492f42f600 ARID=8c3b33c8-751b-4382-b6d8-5b5e1fb04797 \\u003e GW.Http.CallContext/8c3b33c8-751b-4382-b6d8-5b5e1fb04797 \\u003e GW.ExecuteQuery/b1e0bb17-ad83-46fc-92fc-9d1509f8290e \\u003e DN.FE.ExecuteQuery/1633d9fb-3c87-4c20-8b1a-e67b0bd3dac5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[20:18:32.230]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:18:28.0692542Z);\\nlet _endTime = datetime(2026-08-24T20:18:28.0692547Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4gcjkhxx9dcw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:18:32.2085425Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;02369198-f751-4e34-b6ef-a041051f9254\\\",\\r\\n \\\"activityId\\\": \\\"1599c948-9f39-4778-9b37-6affc214343b\\\",\\r\\n \\\"subActivityId\\\": \\\"1032076f-4e50-437f-a461-4aee07de64e1\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b1877adc-b828-4316-8487-63d92a011239\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02369198-f751-4e34-b6ef-a041051f9254 ARID=1599c948-9f39-4778-9b37-6affc214343b \\u003e GW.Http.CallContext/1599c948-9f39-4778-9b37-6affc214343b \\u003e GW.ExecuteQuery/b1877adc-b828-4316-8487-63d92a011239 \\u003e DN.FE.ExecuteQuery/1032076f-4e50-437f-a461-4aee07de64e1)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:18:32.289]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:18:28.0692542Z);\\nlet _endTime = datetime(2026-08-24T20:18:28.0692547Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4gcjkhxx9dcw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:18:32.2085425Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;02369198-f751-4e34-b6ef-a041051f9254\\\",\\r\\n \\\"activityId\\\": \\\"1599c948-9f39-4778-9b37-6affc214343b\\\",\\r\\n \\\"subActivityId\\\": \\\"1032076f-4e50-437f-a461-4aee07de64e1\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b1877adc-b828-4316-8487-63d92a011239\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02369198-f751-4e34-b6ef-a041051f9254 ARID=1599c948-9f39-4778-9b37-6affc214343b \\u003e GW.Http.CallContext/1599c948-9f39-4778-9b37-6affc214343b \\u003e GW.ExecuteQuery/b1877adc-b828-4316-8487-63d92a011239 \\u003e DN.FE.ExecuteQuery/1032076f-4e50-437f-a461-4aee07de64e1)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[20:18:43.645]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:18:38.7262370Z);\\nlet _endTime = datetime(2026-08-24T20:18:38.7262374Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4gcjkhxx9dcw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:18:43.6200013Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0e5312a1-dc13-4b0b-b757-efc107708f9a\\\",\\r\\n \\\"activityId\\\": \\\"493afc98-c82c-491d-bad6-b6399d34f292\\\",\\r\\n \\\"subActivityId\\\": \\\"2090c06b-600c-449f-a94f-d27a4962de6b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d11d9b18-a372-4c5e-9e48-09784e50e381\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0e5312a1-dc13-4b0b-b757-efc107708f9a ARID=493afc98-c82c-491d-bad6-b6399d34f292 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8cc66c09-107e-42be-8f29-290d0f6a372a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8cc66c09-107e-42be-8f29-290d0f6a372a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/d11d9b18-a372-4c5e-9e48-09784e50e381 \\u003e DN.FE.ExecuteQuery/2090c06b-600c-449f-a94f-d27a4962de6b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:18:43.749]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:18:38.7262370Z);\\nlet _endTime = datetime(2026-08-24T20:18:38.7262374Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4gcjkhxx9dcw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:18:43.6200013Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0e5312a1-dc13-4b0b-b757-efc107708f9a\\\",\\r\\n \\\"activityId\\\": \\\"493afc98-c82c-491d-bad6-b6399d34f292\\\",\\r\\n \\\"subActivityId\\\": \\\"2090c06b-600c-449f-a94f-d27a4962de6b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d11d9b18-a372-4c5e-9e48-09784e50e381\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0e5312a1-dc13-4b0b-b757-efc107708f9a ARID=493afc98-c82c-491d-bad6-b6399d34f292 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8cc66c09-107e-42be-8f29-290d0f6a372a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8cc66c09-107e-42be-8f29-290d0f6a372a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/d11d9b18-a372-4c5e-9e48-09784e50e381 \\u003e DN.FE.ExecuteQuery/2090c06b-600c-449f-a94f-d27a4962de6b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00068eed0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[20:18:27.943]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:18:22.8770987Z);\\nlet _endTime = datetime(2026-08-24T20:18:22.8770991Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4gcjkhxx9dcw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:18:27.9010465Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;601b8990-928e-4bd5-baaf-10492f42f600\\\",\\r\\n \\\"activityId\\\": \\\"8c3b33c8-751b-4382-b6d8-5b5e1fb04797\\\",\\r\\n \\\"subActivityId\\\": \\\"1633d9fb-3c87-4c20-8b1a-e67b0bd3dac5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b1e0bb17-ad83-46fc-92fc-9d1509f8290e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;601b8990-928e-4bd5-baaf-10492f42f600 ARID=8c3b33c8-751b-4382-b6d8-5b5e1fb04797 \\u003e GW.Http.CallContext/8c3b33c8-751b-4382-b6d8-5b5e1fb04797 \\u003e GW.ExecuteQuery/b1e0bb17-ad83-46fc-92fc-9d1509f8290e \\u003e DN.FE.ExecuteQuery/1633d9fb-3c87-4c20-8b1a-e67b0bd3dac5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:18:28.002]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:18:22.8770987Z);\\nlet _endTime = datetime(2026-08-24T20:18:22.8770991Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4gcjkhxx9dcw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:18:27.9010465Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;601b8990-928e-4bd5-baaf-10492f42f600\\\",\\r\\n \\\"activityId\\\": \\\"8c3b33c8-751b-4382-b6d8-5b5e1fb04797\\\",\\r\\n \\\"subActivityId\\\": \\\"1633d9fb-3c87-4c20-8b1a-e67b0bd3dac5\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b1e0bb17-ad83-46fc-92fc-9d1509f8290e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;601b8990-928e-4bd5-baaf-10492f42f600 ARID=8c3b33c8-751b-4382-b6d8-5b5e1fb04797 \\u003e GW.Http.CallContext/8c3b33c8-751b-4382-b6d8-5b5e1fb04797 \\u003e GW.ExecuteQuery/b1e0bb17-ad83-46fc-92fc-9d1509f8290e \\u003e DN.FE.ExecuteQuery/1633d9fb-3c87-4c20-8b1a-e67b0bd3dac5)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[20:18:32.230]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:18:28.0692542Z);\\nlet _endTime = datetime(2026-08-24T20:18:28.0692547Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4gcjkhxx9dcw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:18:32.2085425Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;02369198-f751-4e34-b6ef-a041051f9254\\\",\\r\\n \\\"activityId\\\": \\\"1599c948-9f39-4778-9b37-6affc214343b\\\",\\r\\n \\\"subActivityId\\\": \\\"1032076f-4e50-437f-a461-4aee07de64e1\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b1877adc-b828-4316-8487-63d92a011239\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02369198-f751-4e34-b6ef-a041051f9254 ARID=1599c948-9f39-4778-9b37-6affc214343b \\u003e GW.Http.CallContext/1599c948-9f39-4778-9b37-6affc214343b \\u003e GW.ExecuteQuery/b1877adc-b828-4316-8487-63d92a011239 \\u003e DN.FE.ExecuteQuery/1032076f-4e50-437f-a461-4aee07de64e1)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:18:32.289]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:18:28.0692542Z);\\nlet _endTime = datetime(2026-08-24T20:18:28.0692547Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4gcjkhxx9dcw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:18:32.2085425Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;02369198-f751-4e34-b6ef-a041051f9254\\\",\\r\\n \\\"activityId\\\": \\\"1599c948-9f39-4778-9b37-6affc214343b\\\",\\r\\n \\\"subActivityId\\\": \\\"1032076f-4e50-437f-a461-4aee07de64e1\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b1877adc-b828-4316-8487-63d92a011239\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;02369198-f751-4e34-b6ef-a041051f9254 ARID=1599c948-9f39-4778-9b37-6affc214343b \\u003e GW.Http.CallContext/1599c948-9f39-4778-9b37-6affc214343b \\u003e GW.ExecuteQuery/b1877adc-b828-4316-8487-63d92a011239 \\u003e DN.FE.ExecuteQuery/1032076f-4e50-437f-a461-4aee07de64e1)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[20:18:43.645]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:18:38.7262370Z);\\nlet _endTime = datetime(2026-08-24T20:18:38.7262374Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4gcjkhxx9dcw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:18:43.6200013Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0e5312a1-dc13-4b0b-b757-efc107708f9a\\\",\\r\\n \\\"activityId\\\": \\\"493afc98-c82c-491d-bad6-b6399d34f292\\\",\\r\\n \\\"subActivityId\\\": \\\"2090c06b-600c-449f-a94f-d27a4962de6b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d11d9b18-a372-4c5e-9e48-09784e50e381\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0e5312a1-dc13-4b0b-b757-efc107708f9a ARID=493afc98-c82c-491d-bad6-b6399d34f292 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8cc66c09-107e-42be-8f29-290d0f6a372a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8cc66c09-107e-42be-8f29-290d0f6a372a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/d11d9b18-a372-4c5e-9e48-09784e50e381 \\u003e DN.FE.ExecuteQuery/2090c06b-600c-449f-a94f-d27a4962de6b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:18:43.749]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:18:38.7262370Z);\\nlet _endTime = datetime(2026-08-24T20:18:38.7262374Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-4gcjkhxx9dcw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:18:43.6200013Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0e5312a1-dc13-4b0b-b757-efc107708f9a\\\",\\r\\n \\\"activityId\\\": \\\"493afc98-c82c-491d-bad6-b6399d34f292\\\",\\r\\n \\\"subActivityId\\\": \\\"2090c06b-600c-449f-a94f-d27a4962de6b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d11d9b18-a372-4c5e-9e48-09784e50e381\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0e5312a1-dc13-4b0b-b757-efc107708f9a ARID=493afc98-c82c-491d-bad6-b6399d34f292 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8cc66c09-107e-42be-8f29-290d0f6a372a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/8cc66c09-107e-42be-8f29-290d0f6a372a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/d11d9b18-a372-4c5e-9e48-09784e50e381 \\u003e DN.FE.ExecuteQuery/2090c06b-600c-449f-a94f-d27a4962de6b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963250454302720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6638,"pr_state":"open","pr_sha":"c403afe8d857fa8c76e2da31fc1c9d3952ca11b9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T18:58:00Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4f4cf5d57d6646c5c3f7a9c276f685d1179e0407cae6e40d7a3de33f525dc554","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963250454302720","occurred_at":"2026-08-24T18:58:00Z","failed_at":"e2e","signature_id":"75d7cd315bc777c1079b9374c7a2b4aabd61d42a04817f2523f7ea7dab1326a9","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--6mjzd2t8mqtv/cluster-hs-pre-wv58r8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000668260\u003e: \n failed to create HCP cluster cluster-hs-pre-wv58r8: failed waiting for cluster=\"cluster-hs-pre-wv58r8\" in resourcegroup=\"rg-hs-pre--6mjzd2t8mqtv\" to finish creating: GET https://rp.j4302720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eab453e-b10a-4490-8636-54b57901a948\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eab453e-b10a-4490-8636-54b57901a948\",\n \"name\": \"5eab453e-b10a-4490-8636-54b57901a948\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T19:44:35.517103456Z\",\n \"endTime\": \"2026-08-24T20:03:43.794426957Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--6mjzd2t8mqtv/cluster-hs-pre-wv58r8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000668260\u003e: \n failed to create HCP cluster cluster-hs-pre-wv58r8: failed waiting for cluster=\"cluster-hs-pre-wv58r8\" in resourcegroup=\"rg-hs-pre--6mjzd2t8mqtv\" to finish creating: GET https://rp.j4302720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eab453e-b10a-4490-8636-54b57901a948\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eab453e-b10a-4490-8636-54b57901a948\",\n \"name\": \"5eab453e-b10a-4490-8636-54b57901a948\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T19:44:35.517103456Z\",\n \"endTime\": \"2026-08-24T20:03:43.794426957Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cc23e17fa5e3d27ef907469f6c8c9eca6bd3152288c8d8e95b641a4332511c5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963250454302720","occurred_at":"2026-08-24T18:58:00Z","failed_at":"e2e","signature_id":"1bd7b81142bf5c01b0963c412981b707f78cb8184adf320516339e4aa3a3ce9f","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00084e500\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[20:11:06.365]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:11:03.1104533Z);\\nlet _endTime = datetime(2026-08-24T20:11:03.1104538Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xnm2zp8mkw5p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:11:06.3437566Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b4cbf958-ca9a-43ef-b045-0cee872da282\\\",\\r\\n \\\"activityId\\\": \\\"cfd929d4-6a8a-483d-8797-30fd98f59ea0\\\",\\r\\n \\\"subActivityId\\\": \\\"70f461c7-3c0c-41c5-931e-7af17d9a102d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f45a7835-de0e-40e8-9b65-69f97a873ce5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b4cbf958-ca9a-43ef-b045-0cee872da282 ARID=cfd929d4-6a8a-483d-8797-30fd98f59ea0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/be02d506-c5a8-4381-824a-4e22e72b0fa9 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/be02d506-c5a8-4381-824a-4e22e72b0fa9 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f45a7835-de0e-40e8-9b65-69f97a873ce5 \\u003e DN.FE.ExecuteQuery/70f461c7-3c0c-41c5-931e-7af17d9a102d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:11:06.417]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:11:03.1104533Z);\\nlet _endTime = datetime(2026-08-24T20:11:03.1104538Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xnm2zp8mkw5p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:11:06.3437566Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b4cbf958-ca9a-43ef-b045-0cee872da282\\\",\\r\\n \\\"activityId\\\": \\\"cfd929d4-6a8a-483d-8797-30fd98f59ea0\\\",\\r\\n \\\"subActivityId\\\": \\\"70f461c7-3c0c-41c5-931e-7af17d9a102d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f45a7835-de0e-40e8-9b65-69f97a873ce5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b4cbf958-ca9a-43ef-b045-0cee872da282 ARID=cfd929d4-6a8a-483d-8797-30fd98f59ea0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/be02d506-c5a8-4381-824a-4e22e72b0fa9 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/be02d506-c5a8-4381-824a-4e22e72b0fa9 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f45a7835-de0e-40e8-9b65-69f97a873ce5 \\u003e DN.FE.ExecuteQuery/70f461c7-3c0c-41c5-931e-7af17d9a102d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[20:11:09.781]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:11:06.4896507Z);\\nlet _endTime = datetime(2026-08-24T20:11:06.4896513Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xnm2zp8mkw5p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:11:09.7597259Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b7256059-edf2-4c2f-80f2-3bee9c0305c7\\\",\\r\\n \\\"activityId\\\": \\\"8d86dded-b5f8-4d3b-9e9a-8b2f53a3c3ab\\\",\\r\\n \\\"subActivityId\\\": \\\"ea2d196d-0243-4e19-91e6-6f5f837fa783\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"34771c8b-9434-4012-a5f3-a7296d99e393\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b7256059-edf2-4c2f-80f2-3bee9c0305c7 ARID=8d86dded-b5f8-4d3b-9e9a-8b2f53a3c3ab \\u003e GW.Http.CallContext/8d86dded-b5f8-4d3b-9e9a-8b2f53a3c3ab \\u003e GW.ExecuteQuery/34771c8b-9434-4012-a5f3-a7296d99e393 \\u003e DN.FE.ExecuteQuery/ea2d196d-0243-4e19-91e6-6f5f837fa783)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:11:09.842]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:11:06.4896507Z);\\nlet _endTime = datetime(2026-08-24T20:11:06.4896513Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xnm2zp8mkw5p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:11:09.7597259Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b7256059-edf2-4c2f-80f2-3bee9c0305c7\\\",\\r\\n \\\"activityId\\\": \\\"8d86dded-b5f8-4d3b-9e9a-8b2f53a3c3ab\\\",\\r\\n \\\"subActivityId\\\": \\\"ea2d196d-0243-4e19-91e6-6f5f837fa783\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"34771c8b-9434-4012-a5f3-a7296d99e393\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b7256059-edf2-4c2f-80f2-3bee9c0305c7 ARID=8d86dded-b5f8-4d3b-9e9a-8b2f53a3c3ab \\u003e GW.Http.CallContext/8d86dded-b5f8-4d3b-9e9a-8b2f53a3c3ab \\u003e GW.ExecuteQuery/34771c8b-9434-4012-a5f3-a7296d99e393 \\u003e DN.FE.ExecuteQuery/ea2d196d-0243-4e19-91e6-6f5f837fa783)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[20:11:16.142]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:11:13.0624028Z);\\nlet _endTime = datetime(2026-08-24T20:11:13.0624037Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xnm2zp8mkw5p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:11:16.1213348Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;efeb4b63-c2c4-476e-8686-4e74080c884a\\\",\\r\\n \\\"activityId\\\": \\\"382114ad-bdad-42eb-b0fe-95ae67f19abb\\\",\\r\\n \\\"subActivityId\\\": \\\"7bae5a33-a42c-4b39-bcd3-5b037a769b51\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6f208b47-b7ff-40cd-9704-f6cfae775429\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;efeb4b63-c2c4-476e-8686-4e74080c884a ARID=382114ad-bdad-42eb-b0fe-95ae67f19abb \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c2626d9e-6d97-4479-897b-c39ca1bcc4f6 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c2626d9e-6d97-4479-897b-c39ca1bcc4f6 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6f208b47-b7ff-40cd-9704-f6cfae775429 \\u003e DN.FE.ExecuteQuery/7bae5a33-a42c-4b39-bcd3-5b037a769b51)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:11:16.256]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:11:13.0624028Z);\\nlet _endTime = datetime(2026-08-24T20:11:13.0624037Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xnm2zp8mkw5p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:11:16.1213348Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;efeb4b63-c2c4-476e-8686-4e74080c884a\\\",\\r\\n \\\"activityId\\\": \\\"382114ad-bdad-42eb-b0fe-95ae67f19abb\\\",\\r\\n \\\"subActivityId\\\": \\\"7bae5a33-a42c-4b39-bcd3-5b037a769b51\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6f208b47-b7ff-40cd-9704-f6cfae775429\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;efeb4b63-c2c4-476e-8686-4e74080c884a ARID=382114ad-bdad-42eb-b0fe-95ae67f19abb \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c2626d9e-6d97-4479-897b-c39ca1bcc4f6 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c2626d9e-6d97-4479-897b-c39ca1bcc4f6 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6f208b47-b7ff-40cd-9704-f6cfae775429 \\u003e DN.FE.ExecuteQuery/7bae5a33-a42c-4b39-bcd3-5b037a769b51)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00084e500\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[20:11:06.365]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:11:03.1104533Z);\\nlet _endTime = datetime(2026-08-24T20:11:03.1104538Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xnm2zp8mkw5p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:11:06.3437566Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b4cbf958-ca9a-43ef-b045-0cee872da282\\\",\\r\\n \\\"activityId\\\": \\\"cfd929d4-6a8a-483d-8797-30fd98f59ea0\\\",\\r\\n \\\"subActivityId\\\": \\\"70f461c7-3c0c-41c5-931e-7af17d9a102d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f45a7835-de0e-40e8-9b65-69f97a873ce5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b4cbf958-ca9a-43ef-b045-0cee872da282 ARID=cfd929d4-6a8a-483d-8797-30fd98f59ea0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/be02d506-c5a8-4381-824a-4e22e72b0fa9 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/be02d506-c5a8-4381-824a-4e22e72b0fa9 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f45a7835-de0e-40e8-9b65-69f97a873ce5 \\u003e DN.FE.ExecuteQuery/70f461c7-3c0c-41c5-931e-7af17d9a102d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:11:06.417]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:11:03.1104533Z);\\nlet _endTime = datetime(2026-08-24T20:11:03.1104538Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xnm2zp8mkw5p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:11:06.3437566Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b4cbf958-ca9a-43ef-b045-0cee872da282\\\",\\r\\n \\\"activityId\\\": \\\"cfd929d4-6a8a-483d-8797-30fd98f59ea0\\\",\\r\\n \\\"subActivityId\\\": \\\"70f461c7-3c0c-41c5-931e-7af17d9a102d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f45a7835-de0e-40e8-9b65-69f97a873ce5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b4cbf958-ca9a-43ef-b045-0cee872da282 ARID=cfd929d4-6a8a-483d-8797-30fd98f59ea0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/be02d506-c5a8-4381-824a-4e22e72b0fa9 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/be02d506-c5a8-4381-824a-4e22e72b0fa9 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f45a7835-de0e-40e8-9b65-69f97a873ce5 \\u003e DN.FE.ExecuteQuery/70f461c7-3c0c-41c5-931e-7af17d9a102d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[20:11:09.781]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:11:06.4896507Z);\\nlet _endTime = datetime(2026-08-24T20:11:06.4896513Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xnm2zp8mkw5p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:11:09.7597259Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b7256059-edf2-4c2f-80f2-3bee9c0305c7\\\",\\r\\n \\\"activityId\\\": \\\"8d86dded-b5f8-4d3b-9e9a-8b2f53a3c3ab\\\",\\r\\n \\\"subActivityId\\\": \\\"ea2d196d-0243-4e19-91e6-6f5f837fa783\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"34771c8b-9434-4012-a5f3-a7296d99e393\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b7256059-edf2-4c2f-80f2-3bee9c0305c7 ARID=8d86dded-b5f8-4d3b-9e9a-8b2f53a3c3ab \\u003e GW.Http.CallContext/8d86dded-b5f8-4d3b-9e9a-8b2f53a3c3ab \\u003e GW.ExecuteQuery/34771c8b-9434-4012-a5f3-a7296d99e393 \\u003e DN.FE.ExecuteQuery/ea2d196d-0243-4e19-91e6-6f5f837fa783)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:11:09.842]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:11:06.4896507Z);\\nlet _endTime = datetime(2026-08-24T20:11:06.4896513Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xnm2zp8mkw5p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:11:09.7597259Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b7256059-edf2-4c2f-80f2-3bee9c0305c7\\\",\\r\\n \\\"activityId\\\": \\\"8d86dded-b5f8-4d3b-9e9a-8b2f53a3c3ab\\\",\\r\\n \\\"subActivityId\\\": \\\"ea2d196d-0243-4e19-91e6-6f5f837fa783\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"34771c8b-9434-4012-a5f3-a7296d99e393\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b7256059-edf2-4c2f-80f2-3bee9c0305c7 ARID=8d86dded-b5f8-4d3b-9e9a-8b2f53a3c3ab \\u003e GW.Http.CallContext/8d86dded-b5f8-4d3b-9e9a-8b2f53a3c3ab \\u003e GW.ExecuteQuery/34771c8b-9434-4012-a5f3-a7296d99e393 \\u003e DN.FE.ExecuteQuery/ea2d196d-0243-4e19-91e6-6f5f837fa783)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[20:11:16.142]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:11:13.0624028Z);\\nlet _endTime = datetime(2026-08-24T20:11:13.0624037Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xnm2zp8mkw5p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:11:16.1213348Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;efeb4b63-c2c4-476e-8686-4e74080c884a\\\",\\r\\n \\\"activityId\\\": \\\"382114ad-bdad-42eb-b0fe-95ae67f19abb\\\",\\r\\n \\\"subActivityId\\\": \\\"7bae5a33-a42c-4b39-bcd3-5b037a769b51\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6f208b47-b7ff-40cd-9704-f6cfae775429\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;efeb4b63-c2c4-476e-8686-4e74080c884a ARID=382114ad-bdad-42eb-b0fe-95ae67f19abb \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c2626d9e-6d97-4479-897b-c39ca1bcc4f6 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c2626d9e-6d97-4479-897b-c39ca1bcc4f6 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6f208b47-b7ff-40cd-9704-f6cfae775429 \\u003e DN.FE.ExecuteQuery/7bae5a33-a42c-4b39-bcd3-5b037a769b51)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:11:16.256]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:11:13.0624028Z);\\nlet _endTime = datetime(2026-08-24T20:11:13.0624037Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-xnm2zp8mkw5p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:11:16.1213348Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;efeb4b63-c2c4-476e-8686-4e74080c884a\\\",\\r\\n \\\"activityId\\\": \\\"382114ad-bdad-42eb-b0fe-95ae67f19abb\\\",\\r\\n \\\"subActivityId\\\": \\\"7bae5a33-a42c-4b39-bcd3-5b037a769b51\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6f208b47-b7ff-40cd-9704-f6cfae775429\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;efeb4b63-c2c4-476e-8686-4e74080c884a ARID=382114ad-bdad-42eb-b0fe-95ae67f19abb \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c2626d9e-6d97-4479-897b-c39ca1bcc4f6 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c2626d9e-6d97-4479-897b-c39ca1bcc4f6 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6f208b47-b7ff-40cd-9704-f6cfae775429 \\u003e DN.FE.ExecuteQuery/7bae5a33-a42c-4b39-bcd3-5b037a769b51)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963237305159680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"54bb2db04ddf012dceee9c7b7444143151cf3825","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T18:57:56Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2309d1f643bf2deef5ad989e4688b0847dfd570ecc3ed9c3cfab01d3d58db27e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963237305159680","occurred_at":"2026-08-24T18:57:56Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963205222928384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6584,"pr_state":"open","pr_sha":"c9b84520822a4a33afe34017374746e70acb738e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T18:57:49Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"db23c0c38e7227e8ea63b3cded1321fe478f2c91b81d8093755f40a8c78b2693","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963205222928384","occurred_at":"2026-08-24T18:57:49Z","failed_at":"e2e","signature_id":"9bfada2b49b3fc5245efb1ec3e602fd604d8132a904f08045a044c170e893680","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 601.015s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0007f4420\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[20:27:30.508]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:27:26.9752930Z);\\nlet _endTime = datetime(2026-08-24T20:27:26.9752934Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7cvwsv4nwzpc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:27:30.4873262Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;79d3ac0e-11fc-415c-b3f4-7428bf5d3dd3\\\",\\r\\n \\\"activityId\\\": \\\"8b77f6d0-685b-4580-b228-7fef995a3082\\\",\\r\\n \\\"subActivityId\\\": \\\"dde5fd86-b4e1-4ab1-9e45-83f9419538b9\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"feade8fe-ce29-4ae5-8877-792ab307c6cd\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;79d3ac0e-11fc-415c-b3f4-7428bf5d3dd3 ARID=8b77f6d0-685b-4580-b228-7fef995a3082 \\u003e GW.Http.CallContext/8b77f6d0-685b-4580-b228-7fef995a3082 \\u003e GW.ExecuteQuery/feade8fe-ce29-4ae5-8877-792ab307c6cd \\u003e DN.FE.ExecuteQuery/dde5fd86-b4e1-4ab1-9e45-83f9419538b9)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:27:30.563]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:27:26.9752930Z);\\nlet _endTime = datetime(2026-08-24T20:27:26.9752934Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7cvwsv4nwzpc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:27:30.4873262Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;79d3ac0e-11fc-415c-b3f4-7428bf5d3dd3\\\",\\r\\n \\\"activityId\\\": \\\"8b77f6d0-685b-4580-b228-7fef995a3082\\\",\\r\\n \\\"subActivityId\\\": \\\"dde5fd86-b4e1-4ab1-9e45-83f9419538b9\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"feade8fe-ce29-4ae5-8877-792ab307c6cd\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;79d3ac0e-11fc-415c-b3f4-7428bf5d3dd3 ARID=8b77f6d0-685b-4580-b228-7fef995a3082 \\u003e GW.Http.CallContext/8b77f6d0-685b-4580-b228-7fef995a3082 \\u003e GW.ExecuteQuery/feade8fe-ce29-4ae5-8877-792ab307c6cd \\u003e DN.FE.ExecuteQuery/dde5fd86-b4e1-4ab1-9e45-83f9419538b9)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[20:27:33.858]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:27:30.6324586Z);\\nlet _endTime = datetime(2026-08-24T20:27:30.6324590Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7cvwsv4nwzpc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:27:33.8337270Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c03b81b7-835a-4204-b9d7-986eb63c212e\\\",\\r\\n \\\"activityId\\\": \\\"72d04e18-99e2-42a4-9077-70feb0186d65\\\",\\r\\n \\\"subActivityId\\\": \\\"3bed1316-100e-49f0-8b79-112dca484d33\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2930eef7-01a7-4653-85aa-a04fd7b6171e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c03b81b7-835a-4204-b9d7-986eb63c212e ARID=72d04e18-99e2-42a4-9077-70feb0186d65 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/1c0446ed-dd2a-4622-a43c-42d5895105ed \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/1c0446ed-dd2a-4622-a43c-42d5895105ed \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/2930eef7-01a7-4653-85aa-a04fd7b6171e \\u003e DN.FE.ExecuteQuery/3bed1316-100e-49f0-8b79-112dca484d33)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:27:33.914]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:27:30.6324586Z);\\nlet _endTime = datetime(2026-08-24T20:27:30.6324590Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7cvwsv4nwzpc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:27:33.8337270Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c03b81b7-835a-4204-b9d7-986eb63c212e\\\",\\r\\n \\\"activityId\\\": \\\"72d04e18-99e2-42a4-9077-70feb0186d65\\\",\\r\\n \\\"subActivityId\\\": \\\"3bed1316-100e-49f0-8b79-112dca484d33\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2930eef7-01a7-4653-85aa-a04fd7b6171e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c03b81b7-835a-4204-b9d7-986eb63c212e ARID=72d04e18-99e2-42a4-9077-70feb0186d65 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/1c0446ed-dd2a-4622-a43c-42d5895105ed \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/1c0446ed-dd2a-4622-a43c-42d5895105ed \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/2930eef7-01a7-4653-85aa-a04fd7b6171e \\u003e DN.FE.ExecuteQuery/3bed1316-100e-49f0-8b79-112dca484d33)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[20:27:40.748]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:27:37.4605711Z);\\nlet _endTime = datetime(2026-08-24T20:27:37.4605715Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7cvwsv4nwzpc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:27:40.7261539Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f2096b01-fc59-4a87-a98c-750912083a20\\\",\\r\\n \\\"activityId\\\": \\\"834ce702-d6f1-43ba-8ec3-287e74566d46\\\",\\r\\n \\\"subActivityId\\\": \\\"15c9f489-b0a4-4599-85d4-b7bf5e4ce646\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c5cbfae2-c764-4c2d-a189-4e84f1cad88a\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f2096b01-fc59-4a87-a98c-750912083a20 ARID=834ce702-d6f1-43ba-8ec3-287e74566d46 \\u003e GW.Http.CallContext/834ce702-d6f1-43ba-8ec3-287e74566d46 \\u003e GW.ExecuteQuery/c5cbfae2-c764-4c2d-a189-4e84f1cad88a \\u003e DN.FE.ExecuteQuery/15c9f489-b0a4-4599-85d4-b7bf5e4ce646)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:27:40.846]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:27:37.4605711Z);\\nlet _endTime = datetime(2026-08-24T20:27:37.4605715Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7cvwsv4nwzpc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:27:40.7261539Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f2096b01-fc59-4a87-a98c-750912083a20\\\",\\r\\n \\\"activityId\\\": \\\"834ce702-d6f1-43ba-8ec3-287e74566d46\\\",\\r\\n \\\"subActivityId\\\": \\\"15c9f489-b0a4-4599-85d4-b7bf5e4ce646\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c5cbfae2-c764-4c2d-a189-4e84f1cad88a\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f2096b01-fc59-4a87-a98c-750912083a20 ARID=834ce702-d6f1-43ba-8ec3-287e74566d46 \\u003e GW.Http.CallContext/834ce702-d6f1-43ba-8ec3-287e74566d46 \\u003e GW.ExecuteQuery/c5cbfae2-c764-4c2d-a189-4e84f1cad88a \\u003e DN.FE.ExecuteQuery/15c9f489-b0a4-4599-85d4-b7bf5e4ce646)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 601.015s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0007f4420\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[20:27:30.508]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:27:26.9752930Z);\\nlet _endTime = datetime(2026-08-24T20:27:26.9752934Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7cvwsv4nwzpc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:27:30.4873262Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;79d3ac0e-11fc-415c-b3f4-7428bf5d3dd3\\\",\\r\\n \\\"activityId\\\": \\\"8b77f6d0-685b-4580-b228-7fef995a3082\\\",\\r\\n \\\"subActivityId\\\": \\\"dde5fd86-b4e1-4ab1-9e45-83f9419538b9\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"feade8fe-ce29-4ae5-8877-792ab307c6cd\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;79d3ac0e-11fc-415c-b3f4-7428bf5d3dd3 ARID=8b77f6d0-685b-4580-b228-7fef995a3082 \\u003e GW.Http.CallContext/8b77f6d0-685b-4580-b228-7fef995a3082 \\u003e GW.ExecuteQuery/feade8fe-ce29-4ae5-8877-792ab307c6cd \\u003e DN.FE.ExecuteQuery/dde5fd86-b4e1-4ab1-9e45-83f9419538b9)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:27:30.563]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:27:26.9752930Z);\\nlet _endTime = datetime(2026-08-24T20:27:26.9752934Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7cvwsv4nwzpc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:27:30.4873262Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;79d3ac0e-11fc-415c-b3f4-7428bf5d3dd3\\\",\\r\\n \\\"activityId\\\": \\\"8b77f6d0-685b-4580-b228-7fef995a3082\\\",\\r\\n \\\"subActivityId\\\": \\\"dde5fd86-b4e1-4ab1-9e45-83f9419538b9\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"feade8fe-ce29-4ae5-8877-792ab307c6cd\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;79d3ac0e-11fc-415c-b3f4-7428bf5d3dd3 ARID=8b77f6d0-685b-4580-b228-7fef995a3082 \\u003e GW.Http.CallContext/8b77f6d0-685b-4580-b228-7fef995a3082 \\u003e GW.ExecuteQuery/feade8fe-ce29-4ae5-8877-792ab307c6cd \\u003e DN.FE.ExecuteQuery/dde5fd86-b4e1-4ab1-9e45-83f9419538b9)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[20:27:33.858]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:27:30.6324586Z);\\nlet _endTime = datetime(2026-08-24T20:27:30.6324590Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7cvwsv4nwzpc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:27:33.8337270Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c03b81b7-835a-4204-b9d7-986eb63c212e\\\",\\r\\n \\\"activityId\\\": \\\"72d04e18-99e2-42a4-9077-70feb0186d65\\\",\\r\\n \\\"subActivityId\\\": \\\"3bed1316-100e-49f0-8b79-112dca484d33\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2930eef7-01a7-4653-85aa-a04fd7b6171e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c03b81b7-835a-4204-b9d7-986eb63c212e ARID=72d04e18-99e2-42a4-9077-70feb0186d65 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/1c0446ed-dd2a-4622-a43c-42d5895105ed \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/1c0446ed-dd2a-4622-a43c-42d5895105ed \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/2930eef7-01a7-4653-85aa-a04fd7b6171e \\u003e DN.FE.ExecuteQuery/3bed1316-100e-49f0-8b79-112dca484d33)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:27:33.914]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:27:30.6324586Z);\\nlet _endTime = datetime(2026-08-24T20:27:30.6324590Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7cvwsv4nwzpc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:27:33.8337270Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c03b81b7-835a-4204-b9d7-986eb63c212e\\\",\\r\\n \\\"activityId\\\": \\\"72d04e18-99e2-42a4-9077-70feb0186d65\\\",\\r\\n \\\"subActivityId\\\": \\\"3bed1316-100e-49f0-8b79-112dca484d33\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2930eef7-01a7-4653-85aa-a04fd7b6171e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c03b81b7-835a-4204-b9d7-986eb63c212e ARID=72d04e18-99e2-42a4-9077-70feb0186d65 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/1c0446ed-dd2a-4622-a43c-42d5895105ed \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/1c0446ed-dd2a-4622-a43c-42d5895105ed \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/2930eef7-01a7-4653-85aa-a04fd7b6171e \\u003e DN.FE.ExecuteQuery/3bed1316-100e-49f0-8b79-112dca484d33)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[20:27:40.748]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:27:37.4605711Z);\\nlet _endTime = datetime(2026-08-24T20:27:37.4605715Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7cvwsv4nwzpc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:27:40.7261539Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f2096b01-fc59-4a87-a98c-750912083a20\\\",\\r\\n \\\"activityId\\\": \\\"834ce702-d6f1-43ba-8ec3-287e74566d46\\\",\\r\\n \\\"subActivityId\\\": \\\"15c9f489-b0a4-4599-85d4-b7bf5e4ce646\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c5cbfae2-c764-4c2d-a189-4e84f1cad88a\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f2096b01-fc59-4a87-a98c-750912083a20 ARID=834ce702-d6f1-43ba-8ec3-287e74566d46 \\u003e GW.Http.CallContext/834ce702-d6f1-43ba-8ec3-287e74566d46 \\u003e GW.ExecuteQuery/c5cbfae2-c764-4c2d-a189-4e84f1cad88a \\u003e DN.FE.ExecuteQuery/15c9f489-b0a4-4599-85d4-b7bf5e4ce646)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:27:40.846]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:27:37.4605711Z);\\nlet _endTime = datetime(2026-08-24T20:27:37.4605715Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7cvwsv4nwzpc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:27:40.7261539Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f2096b01-fc59-4a87-a98c-750912083a20\\\",\\r\\n \\\"activityId\\\": \\\"834ce702-d6f1-43ba-8ec3-287e74566d46\\\",\\r\\n \\\"subActivityId\\\": \\\"15c9f489-b0a4-4599-85d4-b7bf5e4ce646\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c5cbfae2-c764-4c2d-a189-4e84f1cad88a\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f2096b01-fc59-4a87-a98c-750912083a20 ARID=834ce702-d6f1-43ba-8ec3-287e74566d46 \\u003e GW.Http.CallContext/834ce702-d6f1-43ba-8ec3-287e74566d46 \\u003e GW.ExecuteQuery/c5cbfae2-c764-4c2d-a189-4e84f1cad88a \\u003e DN.FE.ExecuteQuery/15c9f489-b0a4-4599-85d4-b7bf5e4ce646)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6648,"pr_state":"open","pr_sha":"510b6643d5747503e0f00a3494c4d7ddd35c44ee","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T18:53:38Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"18c765501783e935543dfbd3cdd3df84575f3395cb47fe3b1efd509e9e7d591e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","failed_at":"e2e","signature_id":"dea6785389a59c2a975d6ddfe69a85996e881ce704015e7a9c41548aeaca0017","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eacb00\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-9vb844mdss2z\" to finish creating: GET https://rp.j4068096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/950ebef7-ebd5-431d-8b1a-af617af56c1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/950ebef7-ebd5-431d-8b1a-af617af56c1e\",\n \"name\": \"950ebef7-ebd5-431d-8b1a-af617af56c1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T19:41:55.60628386Z\",\n \"endTime\": \"2026-08-24T20:01:00.110147026Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eacb00\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-9vb844mdss2z\" to finish creating: GET https://rp.j4068096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/950ebef7-ebd5-431d-8b1a-af617af56c1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/950ebef7-ebd5-431d-8b1a-af617af56c1e\",\n \"name\": \"950ebef7-ebd5-431d-8b1a-af617af56c1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T19:41:55.60628386Z\",\n \"endTime\": \"2026-08-24T20:01:00.110147026Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9150e1da0e2323c3760bc2d3f4a2987280cfdb267b33948ce66ddae43b72eaa4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","failed_at":"e2e","signature_id":"c742492599a708bf77717b7e4360828e91602b79b9e3f94c759bffe66f0abf90","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001978690\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[20:06:16.734]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:06:13.3553172Z);\\nlet _endTime = datetime(2026-08-24T20:06:13.3553175Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9p7d8t7v9xz6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:06:16.7133479Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;34da49f1-b8c4-423d-9a0d-9594ca2081bc\\\",\\r\\n \\\"activityId\\\": \\\"2f032998-f59c-4ae2-a694-38cc61f6ea94\\\",\\r\\n \\\"subActivityId\\\": \\\"554417e1-fc52-46f3-b197-f82713ea9e34\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7e75cd47-6911-43a3-a862-d633327c1574\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;34da49f1-b8c4-423d-9a0d-9594ca2081bc ARID=2f032998-f59c-4ae2-a694-38cc61f6ea94 \\u003e GW.Http.CallContext/2f032998-f59c-4ae2-a694-38cc61f6ea94 \\u003e GW.ExecuteQuery/7e75cd47-6911-43a3-a862-d633327c1574 \\u003e DN.FE.ExecuteQuery/554417e1-fc52-46f3-b197-f82713ea9e34)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:06:16.786]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:06:13.3553172Z);\\nlet _endTime = datetime(2026-08-24T20:06:13.3553175Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9p7d8t7v9xz6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:06:16.7133479Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;34da49f1-b8c4-423d-9a0d-9594ca2081bc\\\",\\r\\n \\\"activityId\\\": \\\"2f032998-f59c-4ae2-a694-38cc61f6ea94\\\",\\r\\n \\\"subActivityId\\\": \\\"554417e1-fc52-46f3-b197-f82713ea9e34\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7e75cd47-6911-43a3-a862-d633327c1574\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;34da49f1-b8c4-423d-9a0d-9594ca2081bc ARID=2f032998-f59c-4ae2-a694-38cc61f6ea94 \\u003e GW.Http.CallContext/2f032998-f59c-4ae2-a694-38cc61f6ea94 \\u003e GW.ExecuteQuery/7e75cd47-6911-43a3-a862-d633327c1574 \\u003e DN.FE.ExecuteQuery/554417e1-fc52-46f3-b197-f82713ea9e34)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[20:06:20.207]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:06:16.8552371Z);\\nlet _endTime = datetime(2026-08-24T20:06:16.8552374Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9p7d8t7v9xz6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:06:20.1847522Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8f097191-494f-4ed8-bac2-68eee8e5ff98\\\",\\r\\n \\\"activityId\\\": \\\"f091c01f-7b20-44b8-b04f-fb0467df4e03\\\",\\r\\n \\\"subActivityId\\\": \\\"7026bbd3-2c23-4814-a331-0658c9a9b209\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c2506c2f-7658-4205-bf77-f1c430893684\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8f097191-494f-4ed8-bac2-68eee8e5ff98 ARID=f091c01f-7b20-44b8-b04f-fb0467df4e03 \\u003e GW.Http.CallContext/f091c01f-7b20-44b8-b04f-fb0467df4e03 \\u003e GW.ExecuteQuery/c2506c2f-7658-4205-bf77-f1c430893684 \\u003e DN.FE.ExecuteQuery/7026bbd3-2c23-4814-a331-0658c9a9b209)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:06:20.258]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:06:16.8552371Z);\\nlet _endTime = datetime(2026-08-24T20:06:16.8552374Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9p7d8t7v9xz6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:06:20.1847522Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8f097191-494f-4ed8-bac2-68eee8e5ff98\\\",\\r\\n \\\"activityId\\\": \\\"f091c01f-7b20-44b8-b04f-fb0467df4e03\\\",\\r\\n \\\"subActivityId\\\": \\\"7026bbd3-2c23-4814-a331-0658c9a9b209\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c2506c2f-7658-4205-bf77-f1c430893684\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8f097191-494f-4ed8-bac2-68eee8e5ff98 ARID=f091c01f-7b20-44b8-b04f-fb0467df4e03 \\u003e GW.Http.CallContext/f091c01f-7b20-44b8-b04f-fb0467df4e03 \\u003e GW.ExecuteQuery/c2506c2f-7658-4205-bf77-f1c430893684 \\u003e DN.FE.ExecuteQuery/7026bbd3-2c23-4814-a331-0658c9a9b209)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[20:06:27.216]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:06:23.8328269Z);\\nlet _endTime = datetime(2026-08-24T20:06:23.8328273Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9p7d8t7v9xz6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:06:27.1947859Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e15ec81f-eef5-49f2-8e23-4fd3c09c50c6\\\",\\r\\n \\\"activityId\\\": \\\"f3d06959-0b4e-4ef2-9dda-5ba6f765681d\\\",\\r\\n \\\"subActivityId\\\": \\\"31d47128-d285-43b5-80f5-22705e153fd2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"782870fe-7874-4d64-93ab-867f2156413c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e15ec81f-eef5-49f2-8e23-4fd3c09c50c6 ARID=f3d06959-0b4e-4ef2-9dda-5ba6f765681d \\u003e GW.Http.CallContext/f3d06959-0b4e-4ef2-9dda-5ba6f765681d \\u003e GW.ExecuteQuery/782870fe-7874-4d64-93ab-867f2156413c \\u003e DN.FE.ExecuteQuery/31d47128-d285-43b5-80f5-22705e153fd2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:06:27.336]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:06:23.8328269Z);\\nlet _endTime = datetime(2026-08-24T20:06:23.8328273Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9p7d8t7v9xz6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:06:27.1947859Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e15ec81f-eef5-49f2-8e23-4fd3c09c50c6\\\",\\r\\n \\\"activityId\\\": \\\"f3d06959-0b4e-4ef2-9dda-5ba6f765681d\\\",\\r\\n \\\"subActivityId\\\": \\\"31d47128-d285-43b5-80f5-22705e153fd2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"782870fe-7874-4d64-93ab-867f2156413c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e15ec81f-eef5-49f2-8e23-4fd3c09c50c6 ARID=f3d06959-0b4e-4ef2-9dda-5ba6f765681d \\u003e GW.Http.CallContext/f3d06959-0b4e-4ef2-9dda-5ba6f765681d \\u003e GW.ExecuteQuery/782870fe-7874-4d64-93ab-867f2156413c \\u003e DN.FE.ExecuteQuery/31d47128-d285-43b5-80f5-22705e153fd2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001978690\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[20:06:16.734]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:06:13.3553172Z);\\nlet _endTime = datetime(2026-08-24T20:06:13.3553175Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9p7d8t7v9xz6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:06:16.7133479Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;34da49f1-b8c4-423d-9a0d-9594ca2081bc\\\",\\r\\n \\\"activityId\\\": \\\"2f032998-f59c-4ae2-a694-38cc61f6ea94\\\",\\r\\n \\\"subActivityId\\\": \\\"554417e1-fc52-46f3-b197-f82713ea9e34\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7e75cd47-6911-43a3-a862-d633327c1574\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;34da49f1-b8c4-423d-9a0d-9594ca2081bc ARID=2f032998-f59c-4ae2-a694-38cc61f6ea94 \\u003e GW.Http.CallContext/2f032998-f59c-4ae2-a694-38cc61f6ea94 \\u003e GW.ExecuteQuery/7e75cd47-6911-43a3-a862-d633327c1574 \\u003e DN.FE.ExecuteQuery/554417e1-fc52-46f3-b197-f82713ea9e34)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:06:16.786]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:06:13.3553172Z);\\nlet _endTime = datetime(2026-08-24T20:06:13.3553175Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9p7d8t7v9xz6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:06:16.7133479Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;34da49f1-b8c4-423d-9a0d-9594ca2081bc\\\",\\r\\n \\\"activityId\\\": \\\"2f032998-f59c-4ae2-a694-38cc61f6ea94\\\",\\r\\n \\\"subActivityId\\\": \\\"554417e1-fc52-46f3-b197-f82713ea9e34\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"7e75cd47-6911-43a3-a862-d633327c1574\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;34da49f1-b8c4-423d-9a0d-9594ca2081bc ARID=2f032998-f59c-4ae2-a694-38cc61f6ea94 \\u003e GW.Http.CallContext/2f032998-f59c-4ae2-a694-38cc61f6ea94 \\u003e GW.ExecuteQuery/7e75cd47-6911-43a3-a862-d633327c1574 \\u003e DN.FE.ExecuteQuery/554417e1-fc52-46f3-b197-f82713ea9e34)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[20:06:20.207]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:06:16.8552371Z);\\nlet _endTime = datetime(2026-08-24T20:06:16.8552374Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9p7d8t7v9xz6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:06:20.1847522Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8f097191-494f-4ed8-bac2-68eee8e5ff98\\\",\\r\\n \\\"activityId\\\": \\\"f091c01f-7b20-44b8-b04f-fb0467df4e03\\\",\\r\\n \\\"subActivityId\\\": \\\"7026bbd3-2c23-4814-a331-0658c9a9b209\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c2506c2f-7658-4205-bf77-f1c430893684\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8f097191-494f-4ed8-bac2-68eee8e5ff98 ARID=f091c01f-7b20-44b8-b04f-fb0467df4e03 \\u003e GW.Http.CallContext/f091c01f-7b20-44b8-b04f-fb0467df4e03 \\u003e GW.ExecuteQuery/c2506c2f-7658-4205-bf77-f1c430893684 \\u003e DN.FE.ExecuteQuery/7026bbd3-2c23-4814-a331-0658c9a9b209)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:06:20.258]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:06:16.8552371Z);\\nlet _endTime = datetime(2026-08-24T20:06:16.8552374Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9p7d8t7v9xz6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:06:20.1847522Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8f097191-494f-4ed8-bac2-68eee8e5ff98\\\",\\r\\n \\\"activityId\\\": \\\"f091c01f-7b20-44b8-b04f-fb0467df4e03\\\",\\r\\n \\\"subActivityId\\\": \\\"7026bbd3-2c23-4814-a331-0658c9a9b209\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c2506c2f-7658-4205-bf77-f1c430893684\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8f097191-494f-4ed8-bac2-68eee8e5ff98 ARID=f091c01f-7b20-44b8-b04f-fb0467df4e03 \\u003e GW.Http.CallContext/f091c01f-7b20-44b8-b04f-fb0467df4e03 \\u003e GW.ExecuteQuery/c2506c2f-7658-4205-bf77-f1c430893684 \\u003e DN.FE.ExecuteQuery/7026bbd3-2c23-4814-a331-0658c9a9b209)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[20:06:27.216]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:06:23.8328269Z);\\nlet _endTime = datetime(2026-08-24T20:06:23.8328273Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9p7d8t7v9xz6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:06:27.1947859Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e15ec81f-eef5-49f2-8e23-4fd3c09c50c6\\\",\\r\\n \\\"activityId\\\": \\\"f3d06959-0b4e-4ef2-9dda-5ba6f765681d\\\",\\r\\n \\\"subActivityId\\\": \\\"31d47128-d285-43b5-80f5-22705e153fd2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"782870fe-7874-4d64-93ab-867f2156413c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e15ec81f-eef5-49f2-8e23-4fd3c09c50c6 ARID=f3d06959-0b4e-4ef2-9dda-5ba6f765681d \\u003e GW.Http.CallContext/f3d06959-0b4e-4ef2-9dda-5ba6f765681d \\u003e GW.ExecuteQuery/782870fe-7874-4d64-93ab-867f2156413c \\u003e DN.FE.ExecuteQuery/31d47128-d285-43b5-80f5-22705e153fd2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[20:06:27.336]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T20:06:23.8328269Z);\\nlet _endTime = datetime(2026-08-24T20:06:23.8328273Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-9p7d8t7v9xz6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T20:06:27.1947859Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e15ec81f-eef5-49f2-8e23-4fd3c09c50c6\\\",\\r\\n \\\"activityId\\\": \\\"f3d06959-0b4e-4ef2-9dda-5ba6f765681d\\\",\\r\\n \\\"subActivityId\\\": \\\"31d47128-d285-43b5-80f5-22705e153fd2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"782870fe-7874-4d64-93ab-867f2156413c\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e15ec81f-eef5-49f2-8e23-4fd3c09c50c6 ARID=f3d06959-0b4e-4ef2-9dda-5ba6f765681d \\u003e GW.Http.CallContext/f3d06959-0b4e-4ef2-9dda-5ba6f765681d \\u003e GW.ExecuteQuery/782870fe-7874-4d64-93ab-867f2156413c \\u003e DN.FE.ExecuteQuery/31d47128-d285-43b5-80f5-22705e153fd2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"97f16b487c98b1c057a7968f96f8cfc5d31256f1dbc2de2e582b97c9ac79620e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","failed_at":"alert","signature_id":"84936854ba2393da3d6d4a90cbe465eac2bfa99410eb5ae262e1869d118d0d44","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T19:54:32Z\n Ended: 2026-08-24T20:02:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4068096-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd512keke6njcb0ucd6lun18vonei9e-l7h8m1y3e0a2g9h\", pod=\"router-6464bbd6fd-qbsvd\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd512keke6njcb0ucd6lun18vonei9e-l7h8m1y3e0a2g9h/router-6464bbd6fd-qbsvd has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"f5f7d2db23c1ec24702a4147e58505a5b11e68ad670d042d2e687d8e47f5b5f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","failed_at":"alert","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T19:54:46Z\n Ended: 2026-08-24T20:17:49Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j4068096-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"f87a84ec6bac79b8b154d115915b3787e541aa139d55272bc5766e8b1975d749","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","failed_at":"e2e","signature_id":"2021d27012220f14a5eaeb1e574cf0e0cf00596ef715d56d7ab033d1204d0ed5","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f321c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6q97llkgw8zq\" to finish creating: GET https://rp.j4068096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dec0484-f10a-474f-95c1-675630f94b29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dec0484-f10a-474f-95c1-675630f94b29\",\n \"name\": \"2dec0484-f10a-474f-95c1-675630f94b29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T19:41:52.630494716Z\",\n \"endTime\": \"2026-08-24T20:01:00.080950134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f321c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6q97llkgw8zq\" to finish creating: GET https://rp.j4068096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dec0484-f10a-474f-95c1-675630f94b29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dec0484-f10a-474f-95c1-675630f94b29\",\n \"name\": \"2dec0484-f10a-474f-95c1-675630f94b29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T19:41:52.630494716Z\",\n \"endTime\": \"2026-08-24T20:01:00.080950134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091960563839012864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6433,"pr_state":"open","pr_sha":"9ec65054f149d1b7507171a2d613008ef1bc9761","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T18:47:19Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7f2a1c243214d971ed8277240a3b5af47f54dcfe1048d7addd5a82ef7b244b99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091960563839012864","occurred_at":"2026-08-24T18:47:19Z","failed_at":"e2e","signature_id":"ec35c9667d01d0fca7a963152324f60489cc7cd6ddc292c3798abbfb8aa51092","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00110c3b0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[19:51:58.031]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:51:53.2303748Z);\\nlet _endTime = datetime(2026-08-24T19:51:53.2303753Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-28xbv5cwfv2p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:51:58.0038349Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;96128164-ff83-4c34-accc-2d9f110e21f0\\\",\\r\\n \\\"activityId\\\": \\\"8ac87f54-8740-4507-acac-3a376b12846f\\\",\\r\\n \\\"subActivityId\\\": \\\"dbca726b-b9d8-48a5-8295-fd974a28ccf7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"528aab99-b3bf-4f9e-b456-a62d64410d61\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;96128164-ff83-4c34-accc-2d9f110e21f0 ARID=8ac87f54-8740-4507-acac-3a376b12846f \\u003e GW.Http.CallContext/8ac87f54-8740-4507-acac-3a376b12846f \\u003e GW.ExecuteQuery/528aab99-b3bf-4f9e-b456-a62d64410d61 \\u003e DN.FE.ExecuteQuery/dbca726b-b9d8-48a5-8295-fd974a28ccf7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:51:58.178]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:51:53.2303748Z);\\nlet _endTime = datetime(2026-08-24T19:51:53.2303753Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-28xbv5cwfv2p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:51:58.0038349Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;96128164-ff83-4c34-accc-2d9f110e21f0\\\",\\r\\n \\\"activityId\\\": \\\"8ac87f54-8740-4507-acac-3a376b12846f\\\",\\r\\n \\\"subActivityId\\\": \\\"dbca726b-b9d8-48a5-8295-fd974a28ccf7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"528aab99-b3bf-4f9e-b456-a62d64410d61\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;96128164-ff83-4c34-accc-2d9f110e21f0 ARID=8ac87f54-8740-4507-acac-3a376b12846f \\u003e GW.Http.CallContext/8ac87f54-8740-4507-acac-3a376b12846f \\u003e GW.ExecuteQuery/528aab99-b3bf-4f9e-b456-a62d64410d61 \\u003e DN.FE.ExecuteQuery/dbca726b-b9d8-48a5-8295-fd974a28ccf7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[19:52:03.588]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:51:58.4811848Z);\\nlet _endTime = datetime(2026-08-24T19:51:58.4811858Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-28xbv5cwfv2p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:52:03.5678217Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;af1b37d1-40cf-4cf5-8093-fced46cb4c96\\\",\\r\\n \\\"activityId\\\": \\\"d51c8bb8-5365-4422-9f3d-bbc9cca218b5\\\",\\r\\n \\\"subActivityId\\\": \\\"6975476d-10b4-451b-b424-cf3649dfd75d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8d2189a9-3ed5-4f43-814e-a113dc56eb7e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;af1b37d1-40cf-4cf5-8093-fced46cb4c96 ARID=d51c8bb8-5365-4422-9f3d-bbc9cca218b5 \\u003e GW.Http.CallContext/d51c8bb8-5365-4422-9f3d-bbc9cca218b5 \\u003e GW.ExecuteQuery/8d2189a9-3ed5-4f43-814e-a113dc56eb7e \\u003e DN.FE.ExecuteQuery/6975476d-10b4-451b-b424-cf3649dfd75d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:52:03.647]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:51:58.4811848Z);\\nlet _endTime = datetime(2026-08-24T19:51:58.4811858Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-28xbv5cwfv2p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:52:03.5678217Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;af1b37d1-40cf-4cf5-8093-fced46cb4c96\\\",\\r\\n \\\"activityId\\\": \\\"d51c8bb8-5365-4422-9f3d-bbc9cca218b5\\\",\\r\\n \\\"subActivityId\\\": \\\"6975476d-10b4-451b-b424-cf3649dfd75d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8d2189a9-3ed5-4f43-814e-a113dc56eb7e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;af1b37d1-40cf-4cf5-8093-fced46cb4c96 ARID=d51c8bb8-5365-4422-9f3d-bbc9cca218b5 \\u003e GW.Http.CallContext/d51c8bb8-5365-4422-9f3d-bbc9cca218b5 \\u003e GW.ExecuteQuery/8d2189a9-3ed5-4f43-814e-a113dc56eb7e \\u003e DN.FE.ExecuteQuery/6975476d-10b4-451b-b424-cf3649dfd75d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[19:52:14.740]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:52:09.9612940Z);\\nlet _endTime = datetime(2026-08-24T19:52:09.9612946Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-28xbv5cwfv2p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:52:14.7147920Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;99f53a04-bad6-4929-931b-4d38fe5e4afd\\\",\\r\\n \\\"activityId\\\": \\\"b7b4b40b-e6b7-47d5-bc55-11fbc917a523\\\",\\r\\n \\\"subActivityId\\\": \\\"b6e1dd12-84b1-4a1c-8ebf-ce91a8b34dd6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8d568e6d-b055-4816-bdc6-8065304d5792\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;99f53a04-bad6-4929-931b-4d38fe5e4afd ARID=b7b4b40b-e6b7-47d5-bc55-11fbc917a523 \\u003e GW.Http.CallContext/b7b4b40b-e6b7-47d5-bc55-11fbc917a523 \\u003e GW.ExecuteQuery/8d568e6d-b055-4816-bdc6-8065304d5792 \\u003e DN.FE.ExecuteQuery/b6e1dd12-84b1-4a1c-8ebf-ce91a8b34dd6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:52:14.860]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:52:09.9612940Z);\\nlet _endTime = datetime(2026-08-24T19:52:09.9612946Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-28xbv5cwfv2p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:52:14.7147920Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;99f53a04-bad6-4929-931b-4d38fe5e4afd\\\",\\r\\n \\\"activityId\\\": \\\"b7b4b40b-e6b7-47d5-bc55-11fbc917a523\\\",\\r\\n \\\"subActivityId\\\": \\\"b6e1dd12-84b1-4a1c-8ebf-ce91a8b34dd6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8d568e6d-b055-4816-bdc6-8065304d5792\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;99f53a04-bad6-4929-931b-4d38fe5e4afd ARID=b7b4b40b-e6b7-47d5-bc55-11fbc917a523 \\u003e GW.Http.CallContext/b7b4b40b-e6b7-47d5-bc55-11fbc917a523 \\u003e GW.ExecuteQuery/8d568e6d-b055-4816-bdc6-8065304d5792 \\u003e DN.FE.ExecuteQuery/b6e1dd12-84b1-4a1c-8ebf-ce91a8b34dd6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00110c3b0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[19:51:58.031]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:51:53.2303748Z);\\nlet _endTime = datetime(2026-08-24T19:51:53.2303753Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-28xbv5cwfv2p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:51:58.0038349Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;96128164-ff83-4c34-accc-2d9f110e21f0\\\",\\r\\n \\\"activityId\\\": \\\"8ac87f54-8740-4507-acac-3a376b12846f\\\",\\r\\n \\\"subActivityId\\\": \\\"dbca726b-b9d8-48a5-8295-fd974a28ccf7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"528aab99-b3bf-4f9e-b456-a62d64410d61\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;96128164-ff83-4c34-accc-2d9f110e21f0 ARID=8ac87f54-8740-4507-acac-3a376b12846f \\u003e GW.Http.CallContext/8ac87f54-8740-4507-acac-3a376b12846f \\u003e GW.ExecuteQuery/528aab99-b3bf-4f9e-b456-a62d64410d61 \\u003e DN.FE.ExecuteQuery/dbca726b-b9d8-48a5-8295-fd974a28ccf7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:51:58.178]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:51:53.2303748Z);\\nlet _endTime = datetime(2026-08-24T19:51:53.2303753Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-28xbv5cwfv2p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:51:58.0038349Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;96128164-ff83-4c34-accc-2d9f110e21f0\\\",\\r\\n \\\"activityId\\\": \\\"8ac87f54-8740-4507-acac-3a376b12846f\\\",\\r\\n \\\"subActivityId\\\": \\\"dbca726b-b9d8-48a5-8295-fd974a28ccf7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"528aab99-b3bf-4f9e-b456-a62d64410d61\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;96128164-ff83-4c34-accc-2d9f110e21f0 ARID=8ac87f54-8740-4507-acac-3a376b12846f \\u003e GW.Http.CallContext/8ac87f54-8740-4507-acac-3a376b12846f \\u003e GW.ExecuteQuery/528aab99-b3bf-4f9e-b456-a62d64410d61 \\u003e DN.FE.ExecuteQuery/dbca726b-b9d8-48a5-8295-fd974a28ccf7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[19:52:03.588]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:51:58.4811848Z);\\nlet _endTime = datetime(2026-08-24T19:51:58.4811858Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-28xbv5cwfv2p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:52:03.5678217Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;af1b37d1-40cf-4cf5-8093-fced46cb4c96\\\",\\r\\n \\\"activityId\\\": \\\"d51c8bb8-5365-4422-9f3d-bbc9cca218b5\\\",\\r\\n \\\"subActivityId\\\": \\\"6975476d-10b4-451b-b424-cf3649dfd75d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8d2189a9-3ed5-4f43-814e-a113dc56eb7e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;af1b37d1-40cf-4cf5-8093-fced46cb4c96 ARID=d51c8bb8-5365-4422-9f3d-bbc9cca218b5 \\u003e GW.Http.CallContext/d51c8bb8-5365-4422-9f3d-bbc9cca218b5 \\u003e GW.ExecuteQuery/8d2189a9-3ed5-4f43-814e-a113dc56eb7e \\u003e DN.FE.ExecuteQuery/6975476d-10b4-451b-b424-cf3649dfd75d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:52:03.647]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:51:58.4811848Z);\\nlet _endTime = datetime(2026-08-24T19:51:58.4811858Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-28xbv5cwfv2p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:52:03.5678217Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;af1b37d1-40cf-4cf5-8093-fced46cb4c96\\\",\\r\\n \\\"activityId\\\": \\\"d51c8bb8-5365-4422-9f3d-bbc9cca218b5\\\",\\r\\n \\\"subActivityId\\\": \\\"6975476d-10b4-451b-b424-cf3649dfd75d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8d2189a9-3ed5-4f43-814e-a113dc56eb7e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;af1b37d1-40cf-4cf5-8093-fced46cb4c96 ARID=d51c8bb8-5365-4422-9f3d-bbc9cca218b5 \\u003e GW.Http.CallContext/d51c8bb8-5365-4422-9f3d-bbc9cca218b5 \\u003e GW.ExecuteQuery/8d2189a9-3ed5-4f43-814e-a113dc56eb7e \\u003e DN.FE.ExecuteQuery/6975476d-10b4-451b-b424-cf3649dfd75d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[19:52:14.740]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:52:09.9612940Z);\\nlet _endTime = datetime(2026-08-24T19:52:09.9612946Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-28xbv5cwfv2p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:52:14.7147920Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;99f53a04-bad6-4929-931b-4d38fe5e4afd\\\",\\r\\n \\\"activityId\\\": \\\"b7b4b40b-e6b7-47d5-bc55-11fbc917a523\\\",\\r\\n \\\"subActivityId\\\": \\\"b6e1dd12-84b1-4a1c-8ebf-ce91a8b34dd6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8d568e6d-b055-4816-bdc6-8065304d5792\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;99f53a04-bad6-4929-931b-4d38fe5e4afd ARID=b7b4b40b-e6b7-47d5-bc55-11fbc917a523 \\u003e GW.Http.CallContext/b7b4b40b-e6b7-47d5-bc55-11fbc917a523 \\u003e GW.ExecuteQuery/8d568e6d-b055-4816-bdc6-8065304d5792 \\u003e DN.FE.ExecuteQuery/b6e1dd12-84b1-4a1c-8ebf-ce91a8b34dd6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:52:14.860]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:52:09.9612940Z);\\nlet _endTime = datetime(2026-08-24T19:52:09.9612946Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-28xbv5cwfv2p'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:52:14.7147920Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;99f53a04-bad6-4929-931b-4d38fe5e4afd\\\",\\r\\n \\\"activityId\\\": \\\"b7b4b40b-e6b7-47d5-bc55-11fbc917a523\\\",\\r\\n \\\"subActivityId\\\": \\\"b6e1dd12-84b1-4a1c-8ebf-ce91a8b34dd6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"8d568e6d-b055-4816-bdc6-8065304d5792\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;99f53a04-bad6-4929-931b-4d38fe5e4afd ARID=b7b4b40b-e6b7-47d5-bc55-11fbc917a523 \\u003e GW.Http.CallContext/b7b4b40b-e6b7-47d5-bc55-11fbc917a523 \\u003e GW.ExecuteQuery/8d568e6d-b055-4816-bdc6-8065304d5792 \\u003e DN.FE.ExecuteQuery/b6e1dd12-84b1-4a1c-8ebf-ce91a8b34dd6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091957267166400512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6232,"pr_state":"open","pr_sha":"cbb67883add3cc85dd65c43e81a6cb584dbf8021","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T18:34:13Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c0884f7e22fabd197d3c4ed5099fe40a7163640c38649b7fb28f11643fdce6b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091957267166400512","occurred_at":"2026-08-24T18:34:13Z","failed_at":"provision","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-08-24T19:10:07.135Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T19:10:08.078Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T19:10:08.961Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6400512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05\ntime=2026-08-24T19:19:10.348Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05/operationStatuses/08584140082768889498\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/cluster-3vizzf2ahtjlc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6400512-mgmt-2/agentPools/userswft1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AllocationFailed\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft1-38184404-vmss'. Please try alternative sizes 'Standard_E16ds_v6' (zones 2, 3, 1), 'Standard_E16ds_v6' (eastus, zones 1, 2, 3), 'Standard_E16ds_v6' (eastus2, zones 2, 1, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AllocationFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"3\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The VM allocation failed due to an internal error. Please retry later or try deploying to a different location.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b55233e1f233919ecc579e6967f76823775cbaf35c5ee37e0a8883b46726c324","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AllocationFailed; detail message The VM allocation failed due to an internal error. Please retry later or try deploying to a different location.; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b55233e1f233919ecc579e6967f76823775cbaf35c5ee37e0a8883b46726c324"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091956227226472448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6504,"pr_state":"open","pr_sha":"af0ab1f037eef8141b97aadd390cc54296b6d1c3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T18:30:02Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"90747704d269cf655ca06bd3c03f0be3a7a8818689362da42ef94bc2c752ee18","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091956227226472448","occurred_at":"2026-08-24T18:30:02Z","failed_at":"e2e","signature_id":"335052cc293f1dcfc9068c799f022fd529e07859d0cea778ae29ff1ed7adcc44","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0010363a0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[19:34:09.673]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:34:06.2000479Z);\\nlet _endTime = datetime(2026-08-24T19:34:06.2000484Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ppdpqfmxnzrc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:34:09.6484778Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d81cfcef-e8ba-48ea-b997-8cf06756c0b3\\\",\\r\\n \\\"activityId\\\": \\\"054f25c0-1fd0-49a9-99e1-26afd3a6d39a\\\",\\r\\n \\\"subActivityId\\\": \\\"a9e57df5-e41b-41f5-b40e-6c264fe0c14d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f615e944-9838-4c4a-a25e-26cdd6965f94\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d81cfcef-e8ba-48ea-b997-8cf06756c0b3 ARID=054f25c0-1fd0-49a9-99e1-26afd3a6d39a \\u003e GW.Http.CallContext/054f25c0-1fd0-49a9-99e1-26afd3a6d39a \\u003e GW.ExecuteQuery/f615e944-9838-4c4a-a25e-26cdd6965f94 \\u003e DN.FE.ExecuteQuery/a9e57df5-e41b-41f5-b40e-6c264fe0c14d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:34:09.728]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:34:06.2000479Z);\\nlet _endTime = datetime(2026-08-24T19:34:06.2000484Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ppdpqfmxnzrc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:34:09.6484778Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d81cfcef-e8ba-48ea-b997-8cf06756c0b3\\\",\\r\\n \\\"activityId\\\": \\\"054f25c0-1fd0-49a9-99e1-26afd3a6d39a\\\",\\r\\n \\\"subActivityId\\\": \\\"a9e57df5-e41b-41f5-b40e-6c264fe0c14d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f615e944-9838-4c4a-a25e-26cdd6965f94\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d81cfcef-e8ba-48ea-b997-8cf06756c0b3 ARID=054f25c0-1fd0-49a9-99e1-26afd3a6d39a \\u003e GW.Http.CallContext/054f25c0-1fd0-49a9-99e1-26afd3a6d39a \\u003e GW.ExecuteQuery/f615e944-9838-4c4a-a25e-26cdd6965f94 \\u003e DN.FE.ExecuteQuery/a9e57df5-e41b-41f5-b40e-6c264fe0c14d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[19:34:12.790]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:34:09.7961135Z);\\nlet _endTime = datetime(2026-08-24T19:34:09.7961139Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ppdpqfmxnzrc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:34:12.7703289Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a00d3cd1-04de-493c-9e83-debb0c5596f9\\\",\\r\\n \\\"activityId\\\": \\\"f6e21dd3-10a7-425a-8be3-897128f4f105\\\",\\r\\n \\\"subActivityId\\\": \\\"94242df0-6fdf-4a93-831e-3661c9dff018\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"880f89f6-1e55-4a64-ad0f-c2a2d66d12dc\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a00d3cd1-04de-493c-9e83-debb0c5596f9 ARID=f6e21dd3-10a7-425a-8be3-897128f4f105 \\u003e GW.Http.CallContext/f6e21dd3-10a7-425a-8be3-897128f4f105 \\u003e GW.ExecuteQuery/880f89f6-1e55-4a64-ad0f-c2a2d66d12dc \\u003e DN.FE.ExecuteQuery/94242df0-6fdf-4a93-831e-3661c9dff018)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:34:12.839]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:34:09.7961135Z);\\nlet _endTime = datetime(2026-08-24T19:34:09.7961139Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ppdpqfmxnzrc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:34:12.7703289Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a00d3cd1-04de-493c-9e83-debb0c5596f9\\\",\\r\\n \\\"activityId\\\": \\\"f6e21dd3-10a7-425a-8be3-897128f4f105\\\",\\r\\n \\\"subActivityId\\\": \\\"94242df0-6fdf-4a93-831e-3661c9dff018\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"880f89f6-1e55-4a64-ad0f-c2a2d66d12dc\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a00d3cd1-04de-493c-9e83-debb0c5596f9 ARID=f6e21dd3-10a7-425a-8be3-897128f4f105 \\u003e GW.Http.CallContext/f6e21dd3-10a7-425a-8be3-897128f4f105 \\u003e GW.ExecuteQuery/880f89f6-1e55-4a64-ad0f-c2a2d66d12dc \\u003e DN.FE.ExecuteQuery/94242df0-6fdf-4a93-831e-3661c9dff018)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[19:34:18.965]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:34:15.9977616Z);\\nlet _endTime = datetime(2026-08-24T19:34:15.9977620Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ppdpqfmxnzrc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:34:18.9443276Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;15af706b-c55b-4e65-86ce-80b53c2c1b4c\\\",\\r\\n \\\"activityId\\\": \\\"3d775bce-17b2-46ea-9bb0-d1bc0ef2f068\\\",\\r\\n \\\"subActivityId\\\": \\\"6e75bb4c-3bcf-4e6f-9051-548216fd4a21\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f26336ef-c50d-4796-9b1c-509b00e04b53\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;15af706b-c55b-4e65-86ce-80b53c2c1b4c ARID=3d775bce-17b2-46ea-9bb0-d1bc0ef2f068 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d52dede4-e619-4aff-ae18-79e397067d8c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d52dede4-e619-4aff-ae18-79e397067d8c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f26336ef-c50d-4796-9b1c-509b00e04b53 \\u003e DN.FE.ExecuteQuery/6e75bb4c-3bcf-4e6f-9051-548216fd4a21)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:34:19.063]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:34:15.9977616Z);\\nlet _endTime = datetime(2026-08-24T19:34:15.9977620Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ppdpqfmxnzrc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:34:18.9443276Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;15af706b-c55b-4e65-86ce-80b53c2c1b4c\\\",\\r\\n \\\"activityId\\\": \\\"3d775bce-17b2-46ea-9bb0-d1bc0ef2f068\\\",\\r\\n \\\"subActivityId\\\": \\\"6e75bb4c-3bcf-4e6f-9051-548216fd4a21\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f26336ef-c50d-4796-9b1c-509b00e04b53\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;15af706b-c55b-4e65-86ce-80b53c2c1b4c ARID=3d775bce-17b2-46ea-9bb0-d1bc0ef2f068 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d52dede4-e619-4aff-ae18-79e397067d8c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d52dede4-e619-4aff-ae18-79e397067d8c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f26336ef-c50d-4796-9b1c-509b00e04b53 \\u003e DN.FE.ExecuteQuery/6e75bb4c-3bcf-4e6f-9051-548216fd4a21)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0010363a0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[19:34:09.673]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:34:06.2000479Z);\\nlet _endTime = datetime(2026-08-24T19:34:06.2000484Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ppdpqfmxnzrc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:34:09.6484778Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d81cfcef-e8ba-48ea-b997-8cf06756c0b3\\\",\\r\\n \\\"activityId\\\": \\\"054f25c0-1fd0-49a9-99e1-26afd3a6d39a\\\",\\r\\n \\\"subActivityId\\\": \\\"a9e57df5-e41b-41f5-b40e-6c264fe0c14d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f615e944-9838-4c4a-a25e-26cdd6965f94\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d81cfcef-e8ba-48ea-b997-8cf06756c0b3 ARID=054f25c0-1fd0-49a9-99e1-26afd3a6d39a \\u003e GW.Http.CallContext/054f25c0-1fd0-49a9-99e1-26afd3a6d39a \\u003e GW.ExecuteQuery/f615e944-9838-4c4a-a25e-26cdd6965f94 \\u003e DN.FE.ExecuteQuery/a9e57df5-e41b-41f5-b40e-6c264fe0c14d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:34:09.728]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:34:06.2000479Z);\\nlet _endTime = datetime(2026-08-24T19:34:06.2000484Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ppdpqfmxnzrc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:34:09.6484778Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;d81cfcef-e8ba-48ea-b997-8cf06756c0b3\\\",\\r\\n \\\"activityId\\\": \\\"054f25c0-1fd0-49a9-99e1-26afd3a6d39a\\\",\\r\\n \\\"subActivityId\\\": \\\"a9e57df5-e41b-41f5-b40e-6c264fe0c14d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f615e944-9838-4c4a-a25e-26cdd6965f94\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d81cfcef-e8ba-48ea-b997-8cf06756c0b3 ARID=054f25c0-1fd0-49a9-99e1-26afd3a6d39a \\u003e GW.Http.CallContext/054f25c0-1fd0-49a9-99e1-26afd3a6d39a \\u003e GW.ExecuteQuery/f615e944-9838-4c4a-a25e-26cdd6965f94 \\u003e DN.FE.ExecuteQuery/a9e57df5-e41b-41f5-b40e-6c264fe0c14d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[19:34:12.790]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:34:09.7961135Z);\\nlet _endTime = datetime(2026-08-24T19:34:09.7961139Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ppdpqfmxnzrc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:34:12.7703289Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a00d3cd1-04de-493c-9e83-debb0c5596f9\\\",\\r\\n \\\"activityId\\\": \\\"f6e21dd3-10a7-425a-8be3-897128f4f105\\\",\\r\\n \\\"subActivityId\\\": \\\"94242df0-6fdf-4a93-831e-3661c9dff018\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"880f89f6-1e55-4a64-ad0f-c2a2d66d12dc\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a00d3cd1-04de-493c-9e83-debb0c5596f9 ARID=f6e21dd3-10a7-425a-8be3-897128f4f105 \\u003e GW.Http.CallContext/f6e21dd3-10a7-425a-8be3-897128f4f105 \\u003e GW.ExecuteQuery/880f89f6-1e55-4a64-ad0f-c2a2d66d12dc \\u003e DN.FE.ExecuteQuery/94242df0-6fdf-4a93-831e-3661c9dff018)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:34:12.839]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:34:09.7961135Z);\\nlet _endTime = datetime(2026-08-24T19:34:09.7961139Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ppdpqfmxnzrc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:34:12.7703289Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;a00d3cd1-04de-493c-9e83-debb0c5596f9\\\",\\r\\n \\\"activityId\\\": \\\"f6e21dd3-10a7-425a-8be3-897128f4f105\\\",\\r\\n \\\"subActivityId\\\": \\\"94242df0-6fdf-4a93-831e-3661c9dff018\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"880f89f6-1e55-4a64-ad0f-c2a2d66d12dc\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a00d3cd1-04de-493c-9e83-debb0c5596f9 ARID=f6e21dd3-10a7-425a-8be3-897128f4f105 \\u003e GW.Http.CallContext/f6e21dd3-10a7-425a-8be3-897128f4f105 \\u003e GW.ExecuteQuery/880f89f6-1e55-4a64-ad0f-c2a2d66d12dc \\u003e DN.FE.ExecuteQuery/94242df0-6fdf-4a93-831e-3661c9dff018)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[19:34:18.965]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:34:15.9977616Z);\\nlet _endTime = datetime(2026-08-24T19:34:15.9977620Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ppdpqfmxnzrc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:34:18.9443276Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;15af706b-c55b-4e65-86ce-80b53c2c1b4c\\\",\\r\\n \\\"activityId\\\": \\\"3d775bce-17b2-46ea-9bb0-d1bc0ef2f068\\\",\\r\\n \\\"subActivityId\\\": \\\"6e75bb4c-3bcf-4e6f-9051-548216fd4a21\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f26336ef-c50d-4796-9b1c-509b00e04b53\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;15af706b-c55b-4e65-86ce-80b53c2c1b4c ARID=3d775bce-17b2-46ea-9bb0-d1bc0ef2f068 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d52dede4-e619-4aff-ae18-79e397067d8c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d52dede4-e619-4aff-ae18-79e397067d8c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f26336ef-c50d-4796-9b1c-509b00e04b53 \\u003e DN.FE.ExecuteQuery/6e75bb4c-3bcf-4e6f-9051-548216fd4a21)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:34:19.063]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:34:15.9977616Z);\\nlet _endTime = datetime(2026-08-24T19:34:15.9977620Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-ppdpqfmxnzrc'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:34:18.9443276Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;15af706b-c55b-4e65-86ce-80b53c2c1b4c\\\",\\r\\n \\\"activityId\\\": \\\"3d775bce-17b2-46ea-9bb0-d1bc0ef2f068\\\",\\r\\n \\\"subActivityId\\\": \\\"6e75bb4c-3bcf-4e6f-9051-548216fd4a21\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f26336ef-c50d-4796-9b1c-509b00e04b53\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;15af706b-c55b-4e65-86ce-80b53c2c1b4c ARID=3d775bce-17b2-46ea-9bb0-d1bc0ef2f068 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d52dede4-e619-4aff-ae18-79e397067d8c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d52dede4-e619-4aff-ae18-79e397067d8c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/f26336ef-c50d-4796-9b1c-509b00e04b53 \\u003e DN.FE.ExecuteQuery/6e75bb4c-3bcf-4e6f-9051-548216fd4a21)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091954836298797056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6504,"pr_state":"open","pr_sha":"af0ab1f037eef8141b97aadd390cc54296b6d1c3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T18:15:46Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"499085cd622f1d004886cb5874322018287198ba03c7d1101ad90f82bf223cf5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091954836298797056","occurred_at":"2026-08-24T18:15:46Z","failed_at":"provision","signature_id":"0ec7efd378b135d7fa85cb93bcc2f610fe36f768df0ac4e2e0fe6122f4092f85","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/management/output (stamp=1)","test_suite":"step graph","failure_text":"time=2026-08-24T18:57:37.594Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 description=\"Step output\\n Kind: ARM\\n Template: ./../dev-infrastructure/templates/output-mgmt.bicep\\n Parameters: ./../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-24T18:57:37.979Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1\ntime=2026-08-24T18:57:38.350Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 deployment=cfd650f4557893b92396842785417ac4f900952cc40f35ef046b64198c66a837 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8797056-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcfd650f4557893b92396842785417ac4f900952cc40f35ef046b64198c66a837\ntime=2026-08-24T18:57:38.400Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8797056-mgmt-1/providers/Microsoft.Resources/deployments/cfd650f4557893b92396842785417ac4f900952cc40f35ef046b64198c66a837/operationStatuses/08584140090272578417\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'cfd650f4557893b92396842785417ac4f900952cc40f35ef046b64198c66a837' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091950955841982464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6661,"pr_state":"open","pr_sha":"dd56f9b0957f8ae29fbdc7e18bc579c52f4542aa","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T18:09:09Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7bce096e608ed2aefc31821f98de8d6867a231b6f258749857668011e911526e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091950955841982464","occurred_at":"2026-08-24T18:09:09Z","failed_at":"e2e","signature_id":"c0cebe2544e94f85956b2aa4e6cb37d39972915968a1824b87954f1d8f2b0278","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 619.631s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0030681a0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[19:23:44.062]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:23:40.1519515Z);\\nlet _endTime = datetime(2026-08-24T19:23:40.1519520Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-rnpv84j6wl7r'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:23:44.0398678Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;3d7a5f03-f3c0-4765-8dfb-97bafe8a515f\\\",\\r\\n \\\"activityId\\\": \\\"8deb662e-4ed5-4295-b951-12b5409e7ea3\\\",\\r\\n \\\"subActivityId\\\": \\\"0cf0153c-3828-4601-9bcc-7a115de17b07\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"5881bf3c-b1e1-47fb-b608-3ae541bc20d5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;3d7a5f03-f3c0-4765-8dfb-97bafe8a515f ARID=8deb662e-4ed5-4295-b951-12b5409e7ea3 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5e0df9ce-766c-4b6f-8970-5b75e7392f9a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5e0df9ce-766c-4b6f-8970-5b75e7392f9a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/5881bf3c-b1e1-47fb-b608-3ae541bc20d5 \\u003e DN.FE.ExecuteQuery/0cf0153c-3828-4601-9bcc-7a115de17b07)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:23:44.117]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:23:40.1519515Z);\\nlet _endTime = datetime(2026-08-24T19:23:40.1519520Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-rnpv84j6wl7r'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:23:44.0398678Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;3d7a5f03-f3c0-4765-8dfb-97bafe8a515f\\\",\\r\\n \\\"activityId\\\": \\\"8deb662e-4ed5-4295-b951-12b5409e7ea3\\\",\\r\\n \\\"subActivityId\\\": \\\"0cf0153c-3828-4601-9bcc-7a115de17b07\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"5881bf3c-b1e1-47fb-b608-3ae541bc20d5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;3d7a5f03-f3c0-4765-8dfb-97bafe8a515f ARID=8deb662e-4ed5-4295-b951-12b5409e7ea3 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5e0df9ce-766c-4b6f-8970-5b75e7392f9a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5e0df9ce-766c-4b6f-8970-5b75e7392f9a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/5881bf3c-b1e1-47fb-b608-3ae541bc20d5 \\u003e DN.FE.ExecuteQuery/0cf0153c-3828-4601-9bcc-7a115de17b07)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[19:23:47.256]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:23:44.1887769Z);\\nlet _endTime = datetime(2026-08-24T19:23:44.1887774Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-rnpv84j6wl7r'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:23:47.2358741Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8baad5fe-2a10-4dfd-9257-59e57e128d07\\\",\\r\\n \\\"activityId\\\": \\\"1827bac2-5232-4e70-8c09-35636e80a789\\\",\\r\\n \\\"subActivityId\\\": \\\"fa1cc2d9-248b-49c8-bc06-1f5f35c373f0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ecdbfa8e-92e7-472a-91fe-d06a9fbd8588\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8baad5fe-2a10-4dfd-9257-59e57e128d07 ARID=1827bac2-5232-4e70-8c09-35636e80a789 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/992406a7-67e7-4944-9ccf-24ecdf840578 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/992406a7-67e7-4944-9ccf-24ecdf840578 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecdbfa8e-92e7-472a-91fe-d06a9fbd8588 \\u003e DN.FE.ExecuteQuery/fa1cc2d9-248b-49c8-bc06-1f5f35c373f0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:23:47.310]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:23:44.1887769Z);\\nlet _endTime = datetime(2026-08-24T19:23:44.1887774Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-rnpv84j6wl7r'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:23:47.2358741Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8baad5fe-2a10-4dfd-9257-59e57e128d07\\\",\\r\\n \\\"activityId\\\": \\\"1827bac2-5232-4e70-8c09-35636e80a789\\\",\\r\\n \\\"subActivityId\\\": \\\"fa1cc2d9-248b-49c8-bc06-1f5f35c373f0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ecdbfa8e-92e7-472a-91fe-d06a9fbd8588\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8baad5fe-2a10-4dfd-9257-59e57e128d07 ARID=1827bac2-5232-4e70-8c09-35636e80a789 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/992406a7-67e7-4944-9ccf-24ecdf840578 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/992406a7-67e7-4944-9ccf-24ecdf840578 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecdbfa8e-92e7-472a-91fe-d06a9fbd8588 \\u003e DN.FE.ExecuteQuery/fa1cc2d9-248b-49c8-bc06-1f5f35c373f0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[19:23:53.957]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:23:50.5451272Z);\\nlet _endTime = datetime(2026-08-24T19:23:50.5451277Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-rnpv84j6wl7r'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:23:53.9300116Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;90f6c849-5516-4655-952a-2d8ba0fadb0e\\\",\\r\\n \\\"activityId\\\": \\\"69d47628-ee22-4c1a-8e44-ac08996ff22b\\\",\\r\\n \\\"subActivityId\\\": \\\"bcf84e1b-d7ce-46c7-a057-04c241df794c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4f989b02-f73a-48b8-9cc5-b3a9e0cd2026\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;90f6c849-5516-4655-952a-2d8ba0fadb0e ARID=69d47628-ee22-4c1a-8e44-ac08996ff22b \\u003e GW.Http.CallContext/69d47628-ee22-4c1a-8e44-ac08996ff22b \\u003e GW.ExecuteQuery/4f989b02-f73a-48b8-9cc5-b3a9e0cd2026 \\u003e DN.FE.ExecuteQuery/bcf84e1b-d7ce-46c7-a057-04c241df794c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:23:54.071]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:23:50.5451272Z);\\nlet _endTime = datetime(2026-08-24T19:23:50.5451277Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-rnpv84j6wl7r'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:23:53.9300116Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;90f6c849-5516-4655-952a-2d8ba0fadb0e\\\",\\r\\n \\\"activityId\\\": \\\"69d47628-ee22-4c1a-8e44-ac08996ff22b\\\",\\r\\n \\\"subActivityId\\\": \\\"bcf84e1b-d7ce-46c7-a057-04c241df794c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4f989b02-f73a-48b8-9cc5-b3a9e0cd2026\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;90f6c849-5516-4655-952a-2d8ba0fadb0e ARID=69d47628-ee22-4c1a-8e44-ac08996ff22b \\u003e GW.Http.CallContext/69d47628-ee22-4c1a-8e44-ac08996ff22b \\u003e GW.ExecuteQuery/4f989b02-f73a-48b8-9cc5-b3a9e0cd2026 \\u003e DN.FE.ExecuteQuery/bcf84e1b-d7ce-46c7-a057-04c241df794c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 619.631s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0030681a0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[19:23:44.062]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:23:40.1519515Z);\\nlet _endTime = datetime(2026-08-24T19:23:40.1519520Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-rnpv84j6wl7r'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:23:44.0398678Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;3d7a5f03-f3c0-4765-8dfb-97bafe8a515f\\\",\\r\\n \\\"activityId\\\": \\\"8deb662e-4ed5-4295-b951-12b5409e7ea3\\\",\\r\\n \\\"subActivityId\\\": \\\"0cf0153c-3828-4601-9bcc-7a115de17b07\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"5881bf3c-b1e1-47fb-b608-3ae541bc20d5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;3d7a5f03-f3c0-4765-8dfb-97bafe8a515f ARID=8deb662e-4ed5-4295-b951-12b5409e7ea3 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5e0df9ce-766c-4b6f-8970-5b75e7392f9a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5e0df9ce-766c-4b6f-8970-5b75e7392f9a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/5881bf3c-b1e1-47fb-b608-3ae541bc20d5 \\u003e DN.FE.ExecuteQuery/0cf0153c-3828-4601-9bcc-7a115de17b07)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:23:44.117]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:23:40.1519515Z);\\nlet _endTime = datetime(2026-08-24T19:23:40.1519520Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-rnpv84j6wl7r'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:23:44.0398678Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;3d7a5f03-f3c0-4765-8dfb-97bafe8a515f\\\",\\r\\n \\\"activityId\\\": \\\"8deb662e-4ed5-4295-b951-12b5409e7ea3\\\",\\r\\n \\\"subActivityId\\\": \\\"0cf0153c-3828-4601-9bcc-7a115de17b07\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"5881bf3c-b1e1-47fb-b608-3ae541bc20d5\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;3d7a5f03-f3c0-4765-8dfb-97bafe8a515f ARID=8deb662e-4ed5-4295-b951-12b5409e7ea3 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5e0df9ce-766c-4b6f-8970-5b75e7392f9a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/5e0df9ce-766c-4b6f-8970-5b75e7392f9a \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/5881bf3c-b1e1-47fb-b608-3ae541bc20d5 \\u003e DN.FE.ExecuteQuery/0cf0153c-3828-4601-9bcc-7a115de17b07)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[19:23:47.256]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:23:44.1887769Z);\\nlet _endTime = datetime(2026-08-24T19:23:44.1887774Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-rnpv84j6wl7r'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:23:47.2358741Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8baad5fe-2a10-4dfd-9257-59e57e128d07\\\",\\r\\n \\\"activityId\\\": \\\"1827bac2-5232-4e70-8c09-35636e80a789\\\",\\r\\n \\\"subActivityId\\\": \\\"fa1cc2d9-248b-49c8-bc06-1f5f35c373f0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ecdbfa8e-92e7-472a-91fe-d06a9fbd8588\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8baad5fe-2a10-4dfd-9257-59e57e128d07 ARID=1827bac2-5232-4e70-8c09-35636e80a789 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/992406a7-67e7-4944-9ccf-24ecdf840578 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/992406a7-67e7-4944-9ccf-24ecdf840578 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecdbfa8e-92e7-472a-91fe-d06a9fbd8588 \\u003e DN.FE.ExecuteQuery/fa1cc2d9-248b-49c8-bc06-1f5f35c373f0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:23:47.310]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:23:44.1887769Z);\\nlet _endTime = datetime(2026-08-24T19:23:44.1887774Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-rnpv84j6wl7r'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:23:47.2358741Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8baad5fe-2a10-4dfd-9257-59e57e128d07\\\",\\r\\n \\\"activityId\\\": \\\"1827bac2-5232-4e70-8c09-35636e80a789\\\",\\r\\n \\\"subActivityId\\\": \\\"fa1cc2d9-248b-49c8-bc06-1f5f35c373f0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ecdbfa8e-92e7-472a-91fe-d06a9fbd8588\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8baad5fe-2a10-4dfd-9257-59e57e128d07 ARID=1827bac2-5232-4e70-8c09-35636e80a789 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/992406a7-67e7-4944-9ccf-24ecdf840578 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/992406a7-67e7-4944-9ccf-24ecdf840578 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ecdbfa8e-92e7-472a-91fe-d06a9fbd8588 \\u003e DN.FE.ExecuteQuery/fa1cc2d9-248b-49c8-bc06-1f5f35c373f0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[19:23:53.957]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:23:50.5451272Z);\\nlet _endTime = datetime(2026-08-24T19:23:50.5451277Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-rnpv84j6wl7r'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:23:53.9300116Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;90f6c849-5516-4655-952a-2d8ba0fadb0e\\\",\\r\\n \\\"activityId\\\": \\\"69d47628-ee22-4c1a-8e44-ac08996ff22b\\\",\\r\\n \\\"subActivityId\\\": \\\"bcf84e1b-d7ce-46c7-a057-04c241df794c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4f989b02-f73a-48b8-9cc5-b3a9e0cd2026\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;90f6c849-5516-4655-952a-2d8ba0fadb0e ARID=69d47628-ee22-4c1a-8e44-ac08996ff22b \\u003e GW.Http.CallContext/69d47628-ee22-4c1a-8e44-ac08996ff22b \\u003e GW.ExecuteQuery/4f989b02-f73a-48b8-9cc5-b3a9e0cd2026 \\u003e DN.FE.ExecuteQuery/bcf84e1b-d7ce-46c7-a057-04c241df794c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:23:54.071]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:23:50.5451272Z);\\nlet _endTime = datetime(2026-08-24T19:23:50.5451277Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-rnpv84j6wl7r'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:23:53.9300116Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;90f6c849-5516-4655-952a-2d8ba0fadb0e\\\",\\r\\n \\\"activityId\\\": \\\"69d47628-ee22-4c1a-8e44-ac08996ff22b\\\",\\r\\n \\\"subActivityId\\\": \\\"bcf84e1b-d7ce-46c7-a057-04c241df794c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"4f989b02-f73a-48b8-9cc5-b3a9e0cd2026\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;90f6c849-5516-4655-952a-2d8ba0fadb0e ARID=69d47628-ee22-4c1a-8e44-ac08996ff22b \\u003e GW.Http.CallContext/69d47628-ee22-4c1a-8e44-ac08996ff22b \\u003e GW.ExecuteQuery/4f989b02-f73a-48b8-9cc5-b3a9e0cd2026 \\u003e DN.FE.ExecuteQuery/bcf84e1b-d7ce-46c7-a057-04c241df794c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091948175697907712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6529,"pr_state":"open","pr_sha":"09ab1aa72405f5244d5d05015c11621d9cbf759a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T17:58:06Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"43c1a612d430a6a85bcd0e9051266369ce1c2f2d3cde14410e4611456a4deea1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091948175697907712","occurred_at":"2026-08-24T17:58:06Z","failed_at":"e2e","signature_id":"2cfd9e8dd04e05dbf25d7770f4e1349a8d7b20bfb77b1b160b57373b94af3940","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.006s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00113a410\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[19:10:20.778]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:10:17.4484437Z);\\nlet _endTime = datetime(2026-08-24T19:10:17.4484442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jz5mh44r448g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:10:20.7536026Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5dc9f478-db70-4e96-8724-131234c12955\\\",\\r\\n \\\"activityId\\\": \\\"7149451f-d11b-433b-9a3b-571a10a45165\\\",\\r\\n \\\"subActivityId\\\": \\\"fb3e90ce-fa8e-4e9c-bfa9-a3976da8056b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6d73438d-0db4-4318-8b8c-12ab62681dba\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5dc9f478-db70-4e96-8724-131234c12955 ARID=7149451f-d11b-433b-9a3b-571a10a45165 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/3efe508e-a599-46c0-ba0a-5405fc13be3c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/3efe508e-a599-46c0-ba0a-5405fc13be3c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6d73438d-0db4-4318-8b8c-12ab62681dba \\u003e DN.FE.ExecuteQuery/fb3e90ce-fa8e-4e9c-bfa9-a3976da8056b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:10:20.833]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:10:17.4484437Z);\\nlet _endTime = datetime(2026-08-24T19:10:17.4484442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jz5mh44r448g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:10:20.7536026Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5dc9f478-db70-4e96-8724-131234c12955\\\",\\r\\n \\\"activityId\\\": \\\"7149451f-d11b-433b-9a3b-571a10a45165\\\",\\r\\n \\\"subActivityId\\\": \\\"fb3e90ce-fa8e-4e9c-bfa9-a3976da8056b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6d73438d-0db4-4318-8b8c-12ab62681dba\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5dc9f478-db70-4e96-8724-131234c12955 ARID=7149451f-d11b-433b-9a3b-571a10a45165 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/3efe508e-a599-46c0-ba0a-5405fc13be3c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/3efe508e-a599-46c0-ba0a-5405fc13be3c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6d73438d-0db4-4318-8b8c-12ab62681dba \\u003e DN.FE.ExecuteQuery/fb3e90ce-fa8e-4e9c-bfa9-a3976da8056b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[19:10:24.316]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:10:20.8951251Z);\\nlet _endTime = datetime(2026-08-24T19:10:20.8951256Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jz5mh44r448g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:10:24.2915194Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;178dc16e-33e1-4eaf-8836-e5b38c257d27\\\",\\r\\n \\\"activityId\\\": \\\"745e2dd1-7af3-4ad2-9908-aea0f029825a\\\",\\r\\n \\\"subActivityId\\\": \\\"6e56d219-d4a4-4bdb-9237-cec7d05218d2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"433a1557-6ae3-4619-9501-8e588d6e9ab9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;178dc16e-33e1-4eaf-8836-e5b38c257d27 ARID=745e2dd1-7af3-4ad2-9908-aea0f029825a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c030aa9b-1740-40eb-adb1-25e2aeb0a8d9 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c030aa9b-1740-40eb-adb1-25e2aeb0a8d9 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/433a1557-6ae3-4619-9501-8e588d6e9ab9 \\u003e DN.FE.ExecuteQuery/6e56d219-d4a4-4bdb-9237-cec7d05218d2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:10:24.372]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:10:20.8951251Z);\\nlet _endTime = datetime(2026-08-24T19:10:20.8951256Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jz5mh44r448g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:10:24.2915194Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;178dc16e-33e1-4eaf-8836-e5b38c257d27\\\",\\r\\n \\\"activityId\\\": \\\"745e2dd1-7af3-4ad2-9908-aea0f029825a\\\",\\r\\n \\\"subActivityId\\\": \\\"6e56d219-d4a4-4bdb-9237-cec7d05218d2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"433a1557-6ae3-4619-9501-8e588d6e9ab9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;178dc16e-33e1-4eaf-8836-e5b38c257d27 ARID=745e2dd1-7af3-4ad2-9908-aea0f029825a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c030aa9b-1740-40eb-adb1-25e2aeb0a8d9 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c030aa9b-1740-40eb-adb1-25e2aeb0a8d9 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/433a1557-6ae3-4619-9501-8e588d6e9ab9 \\u003e DN.FE.ExecuteQuery/6e56d219-d4a4-4bdb-9237-cec7d05218d2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[19:10:30.712]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:10:27.5126036Z);\\nlet _endTime = datetime(2026-08-24T19:10:27.5126041Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jz5mh44r448g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:10:30.6904943Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2d47fae2-119e-4455-b4d0-4950975b7d93\\\",\\r\\n \\\"activityId\\\": \\\"285a1bcb-0b9a-4c59-a17c-c879f622d59f\\\",\\r\\n \\\"subActivityId\\\": \\\"2eebac36-96de-45fd-8914-3686b027b36c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"011666bc-adc9-46a4-a613-43528a4b1482\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2d47fae2-119e-4455-b4d0-4950975b7d93 ARID=285a1bcb-0b9a-4c59-a17c-c879f622d59f \\u003e GW.Http.CallContext/285a1bcb-0b9a-4c59-a17c-c879f622d59f \\u003e GW.ExecuteQuery/011666bc-adc9-46a4-a613-43528a4b1482 \\u003e DN.FE.ExecuteQuery/2eebac36-96de-45fd-8914-3686b027b36c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:10:30.817]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:10:27.5126036Z);\\nlet _endTime = datetime(2026-08-24T19:10:27.5126041Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jz5mh44r448g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:10:30.6904943Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2d47fae2-119e-4455-b4d0-4950975b7d93\\\",\\r\\n \\\"activityId\\\": \\\"285a1bcb-0b9a-4c59-a17c-c879f622d59f\\\",\\r\\n \\\"subActivityId\\\": \\\"2eebac36-96de-45fd-8914-3686b027b36c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"011666bc-adc9-46a4-a613-43528a4b1482\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2d47fae2-119e-4455-b4d0-4950975b7d93 ARID=285a1bcb-0b9a-4c59-a17c-c879f622d59f \\u003e GW.Http.CallContext/285a1bcb-0b9a-4c59-a17c-c879f622d59f \\u003e GW.ExecuteQuery/011666bc-adc9-46a4-a613-43528a4b1482 \\u003e DN.FE.ExecuteQuery/2eebac36-96de-45fd-8914-3686b027b36c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.006s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00113a410\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[19:10:20.778]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:10:17.4484437Z);\\nlet _endTime = datetime(2026-08-24T19:10:17.4484442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jz5mh44r448g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:10:20.7536026Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5dc9f478-db70-4e96-8724-131234c12955\\\",\\r\\n \\\"activityId\\\": \\\"7149451f-d11b-433b-9a3b-571a10a45165\\\",\\r\\n \\\"subActivityId\\\": \\\"fb3e90ce-fa8e-4e9c-bfa9-a3976da8056b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6d73438d-0db4-4318-8b8c-12ab62681dba\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5dc9f478-db70-4e96-8724-131234c12955 ARID=7149451f-d11b-433b-9a3b-571a10a45165 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/3efe508e-a599-46c0-ba0a-5405fc13be3c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/3efe508e-a599-46c0-ba0a-5405fc13be3c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6d73438d-0db4-4318-8b8c-12ab62681dba \\u003e DN.FE.ExecuteQuery/fb3e90ce-fa8e-4e9c-bfa9-a3976da8056b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:10:20.833]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:10:17.4484437Z);\\nlet _endTime = datetime(2026-08-24T19:10:17.4484442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jz5mh44r448g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:10:20.7536026Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5dc9f478-db70-4e96-8724-131234c12955\\\",\\r\\n \\\"activityId\\\": \\\"7149451f-d11b-433b-9a3b-571a10a45165\\\",\\r\\n \\\"subActivityId\\\": \\\"fb3e90ce-fa8e-4e9c-bfa9-a3976da8056b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6d73438d-0db4-4318-8b8c-12ab62681dba\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5dc9f478-db70-4e96-8724-131234c12955 ARID=7149451f-d11b-433b-9a3b-571a10a45165 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/3efe508e-a599-46c0-ba0a-5405fc13be3c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/3efe508e-a599-46c0-ba0a-5405fc13be3c \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/6d73438d-0db4-4318-8b8c-12ab62681dba \\u003e DN.FE.ExecuteQuery/fb3e90ce-fa8e-4e9c-bfa9-a3976da8056b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[19:10:24.316]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:10:20.8951251Z);\\nlet _endTime = datetime(2026-08-24T19:10:20.8951256Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jz5mh44r448g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:10:24.2915194Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;178dc16e-33e1-4eaf-8836-e5b38c257d27\\\",\\r\\n \\\"activityId\\\": \\\"745e2dd1-7af3-4ad2-9908-aea0f029825a\\\",\\r\\n \\\"subActivityId\\\": \\\"6e56d219-d4a4-4bdb-9237-cec7d05218d2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"433a1557-6ae3-4619-9501-8e588d6e9ab9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;178dc16e-33e1-4eaf-8836-e5b38c257d27 ARID=745e2dd1-7af3-4ad2-9908-aea0f029825a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c030aa9b-1740-40eb-adb1-25e2aeb0a8d9 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c030aa9b-1740-40eb-adb1-25e2aeb0a8d9 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/433a1557-6ae3-4619-9501-8e588d6e9ab9 \\u003e DN.FE.ExecuteQuery/6e56d219-d4a4-4bdb-9237-cec7d05218d2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:10:24.372]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:10:20.8951251Z);\\nlet _endTime = datetime(2026-08-24T19:10:20.8951256Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jz5mh44r448g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:10:24.2915194Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;178dc16e-33e1-4eaf-8836-e5b38c257d27\\\",\\r\\n \\\"activityId\\\": \\\"745e2dd1-7af3-4ad2-9908-aea0f029825a\\\",\\r\\n \\\"subActivityId\\\": \\\"6e56d219-d4a4-4bdb-9237-cec7d05218d2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"433a1557-6ae3-4619-9501-8e588d6e9ab9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;178dc16e-33e1-4eaf-8836-e5b38c257d27 ARID=745e2dd1-7af3-4ad2-9908-aea0f029825a \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c030aa9b-1740-40eb-adb1-25e2aeb0a8d9 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/c030aa9b-1740-40eb-adb1-25e2aeb0a8d9 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/433a1557-6ae3-4619-9501-8e588d6e9ab9 \\u003e DN.FE.ExecuteQuery/6e56d219-d4a4-4bdb-9237-cec7d05218d2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[19:10:30.712]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:10:27.5126036Z);\\nlet _endTime = datetime(2026-08-24T19:10:27.5126041Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jz5mh44r448g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:10:30.6904943Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2d47fae2-119e-4455-b4d0-4950975b7d93\\\",\\r\\n \\\"activityId\\\": \\\"285a1bcb-0b9a-4c59-a17c-c879f622d59f\\\",\\r\\n \\\"subActivityId\\\": \\\"2eebac36-96de-45fd-8914-3686b027b36c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"011666bc-adc9-46a4-a613-43528a4b1482\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2d47fae2-119e-4455-b4d0-4950975b7d93 ARID=285a1bcb-0b9a-4c59-a17c-c879f622d59f \\u003e GW.Http.CallContext/285a1bcb-0b9a-4c59-a17c-c879f622d59f \\u003e GW.ExecuteQuery/011666bc-adc9-46a4-a613-43528a4b1482 \\u003e DN.FE.ExecuteQuery/2eebac36-96de-45fd-8914-3686b027b36c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[19:10:30.817]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T19:10:27.5126036Z);\\nlet _endTime = datetime(2026-08-24T19:10:27.5126041Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-jz5mh44r448g'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T19:10:30.6904943Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;2d47fae2-119e-4455-b4d0-4950975b7d93\\\",\\r\\n \\\"activityId\\\": \\\"285a1bcb-0b9a-4c59-a17c-c879f622d59f\\\",\\r\\n \\\"subActivityId\\\": \\\"2eebac36-96de-45fd-8914-3686b027b36c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"011666bc-adc9-46a4-a613-43528a4b1482\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2d47fae2-119e-4455-b4d0-4950975b7d93 ARID=285a1bcb-0b9a-4c59-a17c-c879f622d59f \\u003e GW.Http.CallContext/285a1bcb-0b9a-4c59-a17c-c879f622d59f \\u003e GW.ExecuteQuery/011666bc-adc9-46a4-a613-43528a4b1482 \\u003e DN.FE.ExecuteQuery/2eebac36-96de-45fd-8914-3686b027b36c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091942092610736128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6660,"pr_state":"open","pr_sha":"a660b6e55fc0b5bbac8bc93588a1797519a9c76f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T17:33:55Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"10aeff86b34704df216c8a89dd372ca85ab1e968e4132df467a0bb3a52d40bfb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091942092610736128","occurred_at":"2026-08-24T17:33:55Z","failed_at":"e2e","signature_id":"41ffa53c8fc1675a8e8f11ea4fff86270a85dca8d99ed458236b7319ea0aa4d4","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001b003c0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[18:43:29.806]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:43:26.1018327Z);\\nlet _endTime = datetime(2026-08-24T18:43:26.1018334Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-76lkckgjnhnb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:43:29.7846996Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;9c4baa00-2868-44b4-9825-b777d40ca5d5\\\",\\r\\n \\\"activityId\\\": \\\"7aaddfd7-cd62-422f-9ff8-20bc627c5167\\\",\\r\\n \\\"subActivityId\\\": \\\"8b05892c-23bc-47d2-aec0-9c031e80fc1b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"214720a0-3b18-41d3-91a6-6d3d1da6e215\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;9c4baa00-2868-44b4-9825-b777d40ca5d5 ARID=7aaddfd7-cd62-422f-9ff8-20bc627c5167 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e60780d-ba60-40ae-9636-d9c2055ea7e4 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e60780d-ba60-40ae-9636-d9c2055ea7e4 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/214720a0-3b18-41d3-91a6-6d3d1da6e215 \\u003e DN.FE.ExecuteQuery/8b05892c-23bc-47d2-aec0-9c031e80fc1b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:43:29.857]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:43:26.1018327Z);\\nlet _endTime = datetime(2026-08-24T18:43:26.1018334Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-76lkckgjnhnb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:43:29.7846996Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;9c4baa00-2868-44b4-9825-b777d40ca5d5\\\",\\r\\n \\\"activityId\\\": \\\"7aaddfd7-cd62-422f-9ff8-20bc627c5167\\\",\\r\\n \\\"subActivityId\\\": \\\"8b05892c-23bc-47d2-aec0-9c031e80fc1b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"214720a0-3b18-41d3-91a6-6d3d1da6e215\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;9c4baa00-2868-44b4-9825-b777d40ca5d5 ARID=7aaddfd7-cd62-422f-9ff8-20bc627c5167 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e60780d-ba60-40ae-9636-d9c2055ea7e4 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e60780d-ba60-40ae-9636-d9c2055ea7e4 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/214720a0-3b18-41d3-91a6-6d3d1da6e215 \\u003e DN.FE.ExecuteQuery/8b05892c-23bc-47d2-aec0-9c031e80fc1b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[18:43:33.215]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:43:29.9677918Z);\\nlet _endTime = datetime(2026-08-24T18:43:29.9677923Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-76lkckgjnhnb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:43:33.1912041Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f1a1de98-d02f-4676-8915-b15cadc2c684\\\",\\r\\n \\\"activityId\\\": \\\"5e4cd7bf-9461-4d67-88c2-7821f091c9bf\\\",\\r\\n \\\"subActivityId\\\": \\\"d89873a5-1709-4160-8eb3-30bd79fa6a48\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f78e16b5-4442-42a5-8d91-e127bd8a51c3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f1a1de98-d02f-4676-8915-b15cadc2c684 ARID=5e4cd7bf-9461-4d67-88c2-7821f091c9bf \\u003e GW.Http.CallContext/5e4cd7bf-9461-4d67-88c2-7821f091c9bf \\u003e GW.ExecuteQuery/f78e16b5-4442-42a5-8d91-e127bd8a51c3 \\u003e DN.FE.ExecuteQuery/d89873a5-1709-4160-8eb3-30bd79fa6a48)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:43:33.267]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:43:29.9677918Z);\\nlet _endTime = datetime(2026-08-24T18:43:29.9677923Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-76lkckgjnhnb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:43:33.1912041Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f1a1de98-d02f-4676-8915-b15cadc2c684\\\",\\r\\n \\\"activityId\\\": \\\"5e4cd7bf-9461-4d67-88c2-7821f091c9bf\\\",\\r\\n \\\"subActivityId\\\": \\\"d89873a5-1709-4160-8eb3-30bd79fa6a48\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f78e16b5-4442-42a5-8d91-e127bd8a51c3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f1a1de98-d02f-4676-8915-b15cadc2c684 ARID=5e4cd7bf-9461-4d67-88c2-7821f091c9bf \\u003e GW.Http.CallContext/5e4cd7bf-9461-4d67-88c2-7821f091c9bf \\u003e GW.ExecuteQuery/f78e16b5-4442-42a5-8d91-e127bd8a51c3 \\u003e DN.FE.ExecuteQuery/d89873a5-1709-4160-8eb3-30bd79fa6a48)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[18:43:39.952]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:43:36.6153971Z);\\nlet _endTime = datetime(2026-08-24T18:43:36.6153978Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-76lkckgjnhnb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:43:39.9321378Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;4f80dd2a-0964-46f4-a66c-ec4414df5f49\\\",\\r\\n \\\"activityId\\\": \\\"e32b2723-acb7-4237-9317-38e8576a7c3c\\\",\\r\\n \\\"subActivityId\\\": \\\"7c297192-bc29-4dcc-bd6d-6334b4e363c0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e6a1b5c8-345f-4f20-958a-4b363e13136e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4f80dd2a-0964-46f4-a66c-ec4414df5f49 ARID=e32b2723-acb7-4237-9317-38e8576a7c3c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e1b634d9-4b0f-48ff-bd25-6198c34b9476 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e1b634d9-4b0f-48ff-bd25-6198c34b9476 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e6a1b5c8-345f-4f20-958a-4b363e13136e \\u003e DN.FE.ExecuteQuery/7c297192-bc29-4dcc-bd6d-6334b4e363c0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:43:40.060]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:43:36.6153971Z);\\nlet _endTime = datetime(2026-08-24T18:43:36.6153978Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-76lkckgjnhnb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:43:39.9321378Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;4f80dd2a-0964-46f4-a66c-ec4414df5f49\\\",\\r\\n \\\"activityId\\\": \\\"e32b2723-acb7-4237-9317-38e8576a7c3c\\\",\\r\\n \\\"subActivityId\\\": \\\"7c297192-bc29-4dcc-bd6d-6334b4e363c0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e6a1b5c8-345f-4f20-958a-4b363e13136e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4f80dd2a-0964-46f4-a66c-ec4414df5f49 ARID=e32b2723-acb7-4237-9317-38e8576a7c3c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e1b634d9-4b0f-48ff-bd25-6198c34b9476 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e1b634d9-4b0f-48ff-bd25-6198c34b9476 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e6a1b5c8-345f-4f20-958a-4b363e13136e \\u003e DN.FE.ExecuteQuery/7c297192-bc29-4dcc-bd6d-6334b4e363c0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001b003c0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[18:43:29.806]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:43:26.1018327Z);\\nlet _endTime = datetime(2026-08-24T18:43:26.1018334Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-76lkckgjnhnb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:43:29.7846996Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;9c4baa00-2868-44b4-9825-b777d40ca5d5\\\",\\r\\n \\\"activityId\\\": \\\"7aaddfd7-cd62-422f-9ff8-20bc627c5167\\\",\\r\\n \\\"subActivityId\\\": \\\"8b05892c-23bc-47d2-aec0-9c031e80fc1b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"214720a0-3b18-41d3-91a6-6d3d1da6e215\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;9c4baa00-2868-44b4-9825-b777d40ca5d5 ARID=7aaddfd7-cd62-422f-9ff8-20bc627c5167 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e60780d-ba60-40ae-9636-d9c2055ea7e4 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e60780d-ba60-40ae-9636-d9c2055ea7e4 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/214720a0-3b18-41d3-91a6-6d3d1da6e215 \\u003e DN.FE.ExecuteQuery/8b05892c-23bc-47d2-aec0-9c031e80fc1b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:43:29.857]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:43:26.1018327Z);\\nlet _endTime = datetime(2026-08-24T18:43:26.1018334Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-76lkckgjnhnb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:43:29.7846996Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;9c4baa00-2868-44b4-9825-b777d40ca5d5\\\",\\r\\n \\\"activityId\\\": \\\"7aaddfd7-cd62-422f-9ff8-20bc627c5167\\\",\\r\\n \\\"subActivityId\\\": \\\"8b05892c-23bc-47d2-aec0-9c031e80fc1b\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"214720a0-3b18-41d3-91a6-6d3d1da6e215\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;9c4baa00-2868-44b4-9825-b777d40ca5d5 ARID=7aaddfd7-cd62-422f-9ff8-20bc627c5167 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e60780d-ba60-40ae-9636-d9c2055ea7e4 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/2e60780d-ba60-40ae-9636-d9c2055ea7e4 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/214720a0-3b18-41d3-91a6-6d3d1da6e215 \\u003e DN.FE.ExecuteQuery/8b05892c-23bc-47d2-aec0-9c031e80fc1b)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[18:43:33.215]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:43:29.9677918Z);\\nlet _endTime = datetime(2026-08-24T18:43:29.9677923Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-76lkckgjnhnb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:43:33.1912041Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f1a1de98-d02f-4676-8915-b15cadc2c684\\\",\\r\\n \\\"activityId\\\": \\\"5e4cd7bf-9461-4d67-88c2-7821f091c9bf\\\",\\r\\n \\\"subActivityId\\\": \\\"d89873a5-1709-4160-8eb3-30bd79fa6a48\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f78e16b5-4442-42a5-8d91-e127bd8a51c3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f1a1de98-d02f-4676-8915-b15cadc2c684 ARID=5e4cd7bf-9461-4d67-88c2-7821f091c9bf \\u003e GW.Http.CallContext/5e4cd7bf-9461-4d67-88c2-7821f091c9bf \\u003e GW.ExecuteQuery/f78e16b5-4442-42a5-8d91-e127bd8a51c3 \\u003e DN.FE.ExecuteQuery/d89873a5-1709-4160-8eb3-30bd79fa6a48)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:43:33.267]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:43:29.9677918Z);\\nlet _endTime = datetime(2026-08-24T18:43:29.9677923Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-76lkckgjnhnb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:43:33.1912041Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;f1a1de98-d02f-4676-8915-b15cadc2c684\\\",\\r\\n \\\"activityId\\\": \\\"5e4cd7bf-9461-4d67-88c2-7821f091c9bf\\\",\\r\\n \\\"subActivityId\\\": \\\"d89873a5-1709-4160-8eb3-30bd79fa6a48\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"f78e16b5-4442-42a5-8d91-e127bd8a51c3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;f1a1de98-d02f-4676-8915-b15cadc2c684 ARID=5e4cd7bf-9461-4d67-88c2-7821f091c9bf \\u003e GW.Http.CallContext/5e4cd7bf-9461-4d67-88c2-7821f091c9bf \\u003e GW.ExecuteQuery/f78e16b5-4442-42a5-8d91-e127bd8a51c3 \\u003e DN.FE.ExecuteQuery/d89873a5-1709-4160-8eb3-30bd79fa6a48)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[18:43:39.952]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:43:36.6153971Z);\\nlet _endTime = datetime(2026-08-24T18:43:36.6153978Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-76lkckgjnhnb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:43:39.9321378Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;4f80dd2a-0964-46f4-a66c-ec4414df5f49\\\",\\r\\n \\\"activityId\\\": \\\"e32b2723-acb7-4237-9317-38e8576a7c3c\\\",\\r\\n \\\"subActivityId\\\": \\\"7c297192-bc29-4dcc-bd6d-6334b4e363c0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e6a1b5c8-345f-4f20-958a-4b363e13136e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4f80dd2a-0964-46f4-a66c-ec4414df5f49 ARID=e32b2723-acb7-4237-9317-38e8576a7c3c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e1b634d9-4b0f-48ff-bd25-6198c34b9476 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e1b634d9-4b0f-48ff-bd25-6198c34b9476 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e6a1b5c8-345f-4f20-958a-4b363e13136e \\u003e DN.FE.ExecuteQuery/7c297192-bc29-4dcc-bd6d-6334b4e363c0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:43:40.060]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:43:36.6153971Z);\\nlet _endTime = datetime(2026-08-24T18:43:36.6153978Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-76lkckgjnhnb'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:43:39.9321378Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;4f80dd2a-0964-46f4-a66c-ec4414df5f49\\\",\\r\\n \\\"activityId\\\": \\\"e32b2723-acb7-4237-9317-38e8576a7c3c\\\",\\r\\n \\\"subActivityId\\\": \\\"7c297192-bc29-4dcc-bd6d-6334b4e363c0\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"e6a1b5c8-345f-4f20-958a-4b363e13136e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4f80dd2a-0964-46f4-a66c-ec4414df5f49 ARID=e32b2723-acb7-4237-9317-38e8576a7c3c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e1b634d9-4b0f-48ff-bd25-6198c34b9476 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/e1b634d9-4b0f-48ff-bd25-6198c34b9476 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e6a1b5c8-345f-4f20-958a-4b363e13136e \\u003e DN.FE.ExecuteQuery/7c297192-bc29-4dcc-bd6d-6334b4e363c0)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5958,"pr_state":"open","pr_sha":"8ce3ed6e8d91907caf45e2b3ec8c94ff7141fe0a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T17:22:18Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"53578ee625e0edb17a48a599fe3ca677f561e55838401f630da2a7ed9bd0855f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","failed_at":"e2e","signature_id":"7171e132341694fb10089af1503bd1696a386e558f7ad492e46e056500959d3b","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.208.156:42093-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.208.156:42093-\u003e172.30.0.10:53: i/o timeout","failure_pattern_match":{"status":"clustered","failure_pattern_id":"911f609de0f6334698a9ae03ee21b22f86f10fb03ad2176871cac7bbc4b43a5f","failure_pattern":"failed to select node pool \u003cnodepool\u003e version for \u003cocp-channel\u003e channel: Get \"\u003curl\u003e\": dial tcp: lookup api","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api"}},{"row_id":"53f52f5e46c45209988c632a004bf1db7cdc7f80a33672dd72a1159e6de7acb2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","failed_at":"e2e","signature_id":"19ba2327e177a466c74ef48f9d128288b644237f0c8199be7309ae0bf40f5a82","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00095a160\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-knxnfrhmdzr6\" to finish creating: GET https://rp.j8858240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03cb2605-91e0-443c-a252-ceeedea9d18a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03cb2605-91e0-443c-a252-ceeedea9d18a\",\n \"name\": \"03cb2605-91e0-443c-a252-ceeedea9d18a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T18:16:27.708109185Z\",\n \"endTime\": \"2026-08-24T18:35:33.013328294Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00095a160\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-knxnfrhmdzr6\" to finish creating: GET https://rp.j8858240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03cb2605-91e0-443c-a252-ceeedea9d18a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03cb2605-91e0-443c-a252-ceeedea9d18a\",\n \"name\": \"03cb2605-91e0-443c-a252-ceeedea9d18a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T18:16:27.708109185Z\",\n \"endTime\": \"2026-08-24T18:35:33.013328294Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"88193927f898d9193f9d6eb3a7cf5f777b614e79e9e59c31bf21d7cdd125aea4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","failed_at":"e2e","signature_id":"dec2f42fbc616357b928df0a2e19436696b2e782470cc0447de0e33d58d63917","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000af8160\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[18:38:30.078]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:38:19.1391016Z);\\nlet _endTime = datetime(2026-08-24T18:38:19.1391021Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrs5cwxggm6w'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:38:29.9359819Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;38d751d6-5b8d-4f16-bc3b-6be2306ac850\\\",\\r\\n \\\"activityId\\\": \\\"767d7e36-4234-4eec-a531-c0201506201f\\\",\\r\\n \\\"subActivityId\\\": \\\"c8289c24-c101-48b8-a652-550358625e95\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0333b671-454e-490f-8c0f-0844fe7df9fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;38d751d6-5b8d-4f16-bc3b-6be2306ac850 ARID=767d7e36-4234-4eec-a531-c0201506201f \\u003e GW.Http.CallContext/767d7e36-4234-4eec-a531-c0201506201f \\u003e GW.ExecuteQuery/0333b671-454e-490f-8c0f-0844fe7df9fb \\u003e DN.FE.ExecuteQuery/c8289c24-c101-48b8-a652-550358625e95)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:38:30.386]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:38:19.1391016Z);\\nlet _endTime = datetime(2026-08-24T18:38:19.1391021Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrs5cwxggm6w'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:38:29.9359819Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;38d751d6-5b8d-4f16-bc3b-6be2306ac850\\\",\\r\\n \\\"activityId\\\": \\\"767d7e36-4234-4eec-a531-c0201506201f\\\",\\r\\n \\\"subActivityId\\\": \\\"c8289c24-c101-48b8-a652-550358625e95\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0333b671-454e-490f-8c0f-0844fe7df9fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;38d751d6-5b8d-4f16-bc3b-6be2306ac850 ARID=767d7e36-4234-4eec-a531-c0201506201f \\u003e GW.Http.CallContext/767d7e36-4234-4eec-a531-c0201506201f \\u003e GW.ExecuteQuery/0333b671-454e-490f-8c0f-0844fe7df9fb \\u003e DN.FE.ExecuteQuery/c8289c24-c101-48b8-a652-550358625e95)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[18:38:41.465]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:38:30.4518739Z);\\nlet _endTime = datetime(2026-08-24T18:38:30.4518744Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrs5cwxggm6w'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:38:41.4379865Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;23368c59-3298-486f-b16b-51120c31d56d\\\",\\r\\n \\\"activityId\\\": \\\"adbfc21a-3f67-4173-9cab-6bf8f9eb3ff8\\\",\\r\\n \\\"subActivityId\\\": \\\"7c76994e-b5dc-4137-8df0-91160aa33d94\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"35086d4b-fc80-4580-8f06-30c8a6eab682\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;23368c59-3298-486f-b16b-51120c31d56d ARID=adbfc21a-3f67-4173-9cab-6bf8f9eb3ff8 \\u003e GW.Http.CallContext/adbfc21a-3f67-4173-9cab-6bf8f9eb3ff8 \\u003e GW.ExecuteQuery/35086d4b-fc80-4580-8f06-30c8a6eab682 \\u003e DN.FE.ExecuteQuery/7c76994e-b5dc-4137-8df0-91160aa33d94)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:38:41.522]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:38:30.4518739Z);\\nlet _endTime = datetime(2026-08-24T18:38:30.4518744Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrs5cwxggm6w'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:38:41.4379865Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;23368c59-3298-486f-b16b-51120c31d56d\\\",\\r\\n \\\"activityId\\\": \\\"adbfc21a-3f67-4173-9cab-6bf8f9eb3ff8\\\",\\r\\n \\\"subActivityId\\\": \\\"7c76994e-b5dc-4137-8df0-91160aa33d94\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"35086d4b-fc80-4580-8f06-30c8a6eab682\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;23368c59-3298-486f-b16b-51120c31d56d ARID=adbfc21a-3f67-4173-9cab-6bf8f9eb3ff8 \\u003e GW.Http.CallContext/adbfc21a-3f67-4173-9cab-6bf8f9eb3ff8 \\u003e GW.ExecuteQuery/35086d4b-fc80-4580-8f06-30c8a6eab682 \\u003e DN.FE.ExecuteQuery/7c76994e-b5dc-4137-8df0-91160aa33d94)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[18:39:12.706]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:38:55.3973069Z);\\nlet _endTime = datetime(2026-08-24T18:38:55.3973074Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrs5cwxggm6w'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:39:12.6828025Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;9d8bb29c-908a-4cdf-90e4-8935c99a813c\\\",\\r\\n \\\"activityId\\\": \\\"726ff655-0d5c-46b8-8137-e2ff94407940\\\",\\r\\n \\\"subActivityId\\\": \\\"6b3a4ccd-ce64-4c47-8cae-0bdb0403bb3c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"dc7d5fbb-34c4-4834-97c2-f484d996fa95\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;9d8bb29c-908a-4cdf-90e4-8935c99a813c ARID=726ff655-0d5c-46b8-8137-e2ff94407940 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f19dfb92-f849-40a9-955b-3947ba99aa46 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f19dfb92-f849-40a9-955b-3947ba99aa46 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/dc7d5fbb-34c4-4834-97c2-f484d996fa95 \\u003e DN.FE.ExecuteQuery/6b3a4ccd-ce64-4c47-8cae-0bdb0403bb3c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:39:13.039]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:38:55.3973069Z);\\nlet _endTime = datetime(2026-08-24T18:38:55.3973074Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrs5cwxggm6w'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:39:12.6828025Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;9d8bb29c-908a-4cdf-90e4-8935c99a813c\\\",\\r\\n \\\"activityId\\\": \\\"726ff655-0d5c-46b8-8137-e2ff94407940\\\",\\r\\n \\\"subActivityId\\\": \\\"6b3a4ccd-ce64-4c47-8cae-0bdb0403bb3c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"dc7d5fbb-34c4-4834-97c2-f484d996fa95\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;9d8bb29c-908a-4cdf-90e4-8935c99a813c ARID=726ff655-0d5c-46b8-8137-e2ff94407940 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f19dfb92-f849-40a9-955b-3947ba99aa46 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f19dfb92-f849-40a9-955b-3947ba99aa46 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/dc7d5fbb-34c4-4834-97c2-f484d996fa95 \\u003e DN.FE.ExecuteQuery/6b3a4ccd-ce64-4c47-8cae-0bdb0403bb3c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000af8160\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[18:38:30.078]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:38:19.1391016Z);\\nlet _endTime = datetime(2026-08-24T18:38:19.1391021Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrs5cwxggm6w'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:38:29.9359819Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;38d751d6-5b8d-4f16-bc3b-6be2306ac850\\\",\\r\\n \\\"activityId\\\": \\\"767d7e36-4234-4eec-a531-c0201506201f\\\",\\r\\n \\\"subActivityId\\\": \\\"c8289c24-c101-48b8-a652-550358625e95\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0333b671-454e-490f-8c0f-0844fe7df9fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;38d751d6-5b8d-4f16-bc3b-6be2306ac850 ARID=767d7e36-4234-4eec-a531-c0201506201f \\u003e GW.Http.CallContext/767d7e36-4234-4eec-a531-c0201506201f \\u003e GW.ExecuteQuery/0333b671-454e-490f-8c0f-0844fe7df9fb \\u003e DN.FE.ExecuteQuery/c8289c24-c101-48b8-a652-550358625e95)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:38:30.386]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:38:19.1391016Z);\\nlet _endTime = datetime(2026-08-24T18:38:19.1391021Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrs5cwxggm6w'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:38:29.9359819Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;38d751d6-5b8d-4f16-bc3b-6be2306ac850\\\",\\r\\n \\\"activityId\\\": \\\"767d7e36-4234-4eec-a531-c0201506201f\\\",\\r\\n \\\"subActivityId\\\": \\\"c8289c24-c101-48b8-a652-550358625e95\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0333b671-454e-490f-8c0f-0844fe7df9fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;38d751d6-5b8d-4f16-bc3b-6be2306ac850 ARID=767d7e36-4234-4eec-a531-c0201506201f \\u003e GW.Http.CallContext/767d7e36-4234-4eec-a531-c0201506201f \\u003e GW.ExecuteQuery/0333b671-454e-490f-8c0f-0844fe7df9fb \\u003e DN.FE.ExecuteQuery/c8289c24-c101-48b8-a652-550358625e95)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[18:38:41.465]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:38:30.4518739Z);\\nlet _endTime = datetime(2026-08-24T18:38:30.4518744Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrs5cwxggm6w'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:38:41.4379865Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;23368c59-3298-486f-b16b-51120c31d56d\\\",\\r\\n \\\"activityId\\\": \\\"adbfc21a-3f67-4173-9cab-6bf8f9eb3ff8\\\",\\r\\n \\\"subActivityId\\\": \\\"7c76994e-b5dc-4137-8df0-91160aa33d94\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"35086d4b-fc80-4580-8f06-30c8a6eab682\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;23368c59-3298-486f-b16b-51120c31d56d ARID=adbfc21a-3f67-4173-9cab-6bf8f9eb3ff8 \\u003e GW.Http.CallContext/adbfc21a-3f67-4173-9cab-6bf8f9eb3ff8 \\u003e GW.ExecuteQuery/35086d4b-fc80-4580-8f06-30c8a6eab682 \\u003e DN.FE.ExecuteQuery/7c76994e-b5dc-4137-8df0-91160aa33d94)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:38:41.522]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:38:30.4518739Z);\\nlet _endTime = datetime(2026-08-24T18:38:30.4518744Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrs5cwxggm6w'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:38:41.4379865Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;23368c59-3298-486f-b16b-51120c31d56d\\\",\\r\\n \\\"activityId\\\": \\\"adbfc21a-3f67-4173-9cab-6bf8f9eb3ff8\\\",\\r\\n \\\"subActivityId\\\": \\\"7c76994e-b5dc-4137-8df0-91160aa33d94\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"35086d4b-fc80-4580-8f06-30c8a6eab682\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;23368c59-3298-486f-b16b-51120c31d56d ARID=adbfc21a-3f67-4173-9cab-6bf8f9eb3ff8 \\u003e GW.Http.CallContext/adbfc21a-3f67-4173-9cab-6bf8f9eb3ff8 \\u003e GW.ExecuteQuery/35086d4b-fc80-4580-8f06-30c8a6eab682 \\u003e DN.FE.ExecuteQuery/7c76994e-b5dc-4137-8df0-91160aa33d94)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[18:39:12.706]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:38:55.3973069Z);\\nlet _endTime = datetime(2026-08-24T18:38:55.3973074Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrs5cwxggm6w'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:39:12.6828025Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;9d8bb29c-908a-4cdf-90e4-8935c99a813c\\\",\\r\\n \\\"activityId\\\": \\\"726ff655-0d5c-46b8-8137-e2ff94407940\\\",\\r\\n \\\"subActivityId\\\": \\\"6b3a4ccd-ce64-4c47-8cae-0bdb0403bb3c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"dc7d5fbb-34c4-4834-97c2-f484d996fa95\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;9d8bb29c-908a-4cdf-90e4-8935c99a813c ARID=726ff655-0d5c-46b8-8137-e2ff94407940 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f19dfb92-f849-40a9-955b-3947ba99aa46 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f19dfb92-f849-40a9-955b-3947ba99aa46 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/dc7d5fbb-34c4-4834-97c2-f484d996fa95 \\u003e DN.FE.ExecuteQuery/6b3a4ccd-ce64-4c47-8cae-0bdb0403bb3c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:39:13.039]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:38:55.3973069Z);\\nlet _endTime = datetime(2026-08-24T18:38:55.3973074Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-lrs5cwxggm6w'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:39:12.6828025Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;9d8bb29c-908a-4cdf-90e4-8935c99a813c\\\",\\r\\n \\\"activityId\\\": \\\"726ff655-0d5c-46b8-8137-e2ff94407940\\\",\\r\\n \\\"subActivityId\\\": \\\"6b3a4ccd-ce64-4c47-8cae-0bdb0403bb3c\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"dc7d5fbb-34c4-4834-97c2-f484d996fa95\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;9d8bb29c-908a-4cdf-90e4-8935c99a813c ARID=726ff655-0d5c-46b8-8137-e2ff94407940 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f19dfb92-f849-40a9-955b-3947ba99aa46 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f19dfb92-f849-40a9-955b-3947ba99aa46 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/dc7d5fbb-34c4-4834-97c2-f484d996fa95 \\u003e DN.FE.ExecuteQuery/6b3a4ccd-ce64-4c47-8cae-0bdb0403bb3c)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"d4f1aa0587f71142127559cbd14f41cf8729b8f7876b5305d7b4fc301f997306","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","failed_at":"alert","signature_id":"d6263c32bd22604b284f6288625ad3fcd30c678787956b94fe5bf7e9a8e0f6cc","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T18:29:40Z\n Ended: 2026-08-24T18:48:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8858240-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd3ove2kcm8cumvp5a7nli0cn48b8df-f7o4e8w4g1n8d1y\", pod=\"router-756bb97467-6q2rw\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd3ove2kcm8cumvp5a7nli0cn48b8df-f7o4e8w4g1n8d1y/router-756bb97467-6q2rw has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","911f609de0f6334698a9ae03ee21b22f86f10fb03ad2176871cac7bbc4b43a5f","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932257060130816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6629,"pr_state":"open","pr_sha":"c6d46781a7abcc4c551cdf9edbb601d3aee0cc38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T16:54:39Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3456481f6cb0ac90dd4d7dd64cf83428e597b44af294d21f338a3837d4783a81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932257060130816","occurred_at":"2026-08-24T16:54:39Z","failed_at":"e2e","signature_id":"cff96711d2e8bb5b9f48911a18c760f7b843abef0ee0b1364775c2b0b5f58a49","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001aa20b0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[18:10:24.297]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:10:20.4586249Z);\\nlet _endTime = datetime(2026-08-24T18:10:20.4586253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-52n7z2mfccvl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:10:24.2714273Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;12b2ecfe-09e1-4838-993b-5dcf60d0b406\\\",\\r\\n \\\"activityId\\\": \\\"04908727-e2e1-425c-ac4a-03f0802e4426\\\",\\r\\n \\\"subActivityId\\\": \\\"c2938410-8cf8-48ad-984d-fb727921dbd7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6ec85876-5223-4015-aeb3-32640b5a0ad9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;12b2ecfe-09e1-4838-993b-5dcf60d0b406 ARID=04908727-e2e1-425c-ac4a-03f0802e4426 \\u003e GW.Http.CallContext/04908727-e2e1-425c-ac4a-03f0802e4426 \\u003e GW.ExecuteQuery/6ec85876-5223-4015-aeb3-32640b5a0ad9 \\u003e DN.FE.ExecuteQuery/c2938410-8cf8-48ad-984d-fb727921dbd7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:10:24.356]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:10:20.4586249Z);\\nlet _endTime = datetime(2026-08-24T18:10:20.4586253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-52n7z2mfccvl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:10:24.2714273Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;12b2ecfe-09e1-4838-993b-5dcf60d0b406\\\",\\r\\n \\\"activityId\\\": \\\"04908727-e2e1-425c-ac4a-03f0802e4426\\\",\\r\\n \\\"subActivityId\\\": \\\"c2938410-8cf8-48ad-984d-fb727921dbd7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6ec85876-5223-4015-aeb3-32640b5a0ad9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;12b2ecfe-09e1-4838-993b-5dcf60d0b406 ARID=04908727-e2e1-425c-ac4a-03f0802e4426 \\u003e GW.Http.CallContext/04908727-e2e1-425c-ac4a-03f0802e4426 \\u003e GW.ExecuteQuery/6ec85876-5223-4015-aeb3-32640b5a0ad9 \\u003e DN.FE.ExecuteQuery/c2938410-8cf8-48ad-984d-fb727921dbd7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[18:10:27.782]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:10:24.4180884Z);\\nlet _endTime = datetime(2026-08-24T18:10:24.4180888Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-52n7z2mfccvl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:10:27.7556057Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5c5731ff-386a-4c60-a4ae-7e8099e48b63\\\",\\r\\n \\\"activityId\\\": \\\"ccca268d-b5b0-44b7-bb2c-53f62ce07692\\\",\\r\\n \\\"subActivityId\\\": \\\"929136dd-2380-48b4-9972-3d2d6b49e1e2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"08a8bbb1-cd7f-44af-aa81-2a8d989464b8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5c5731ff-386a-4c60-a4ae-7e8099e48b63 ARID=ccca268d-b5b0-44b7-bb2c-53f62ce07692 \\u003e GW.Http.CallContext/ccca268d-b5b0-44b7-bb2c-53f62ce07692 \\u003e GW.ExecuteQuery/08a8bbb1-cd7f-44af-aa81-2a8d989464b8 \\u003e DN.FE.ExecuteQuery/929136dd-2380-48b4-9972-3d2d6b49e1e2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:10:27.872]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:10:24.4180884Z);\\nlet _endTime = datetime(2026-08-24T18:10:24.4180888Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-52n7z2mfccvl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:10:27.7556057Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5c5731ff-386a-4c60-a4ae-7e8099e48b63\\\",\\r\\n \\\"activityId\\\": \\\"ccca268d-b5b0-44b7-bb2c-53f62ce07692\\\",\\r\\n \\\"subActivityId\\\": \\\"929136dd-2380-48b4-9972-3d2d6b49e1e2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"08a8bbb1-cd7f-44af-aa81-2a8d989464b8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5c5731ff-386a-4c60-a4ae-7e8099e48b63 ARID=ccca268d-b5b0-44b7-bb2c-53f62ce07692 \\u003e GW.Http.CallContext/ccca268d-b5b0-44b7-bb2c-53f62ce07692 \\u003e GW.ExecuteQuery/08a8bbb1-cd7f-44af-aa81-2a8d989464b8 \\u003e DN.FE.ExecuteQuery/929136dd-2380-48b4-9972-3d2d6b49e1e2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[18:10:35.410]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:10:31.5845546Z);\\nlet _endTime = datetime(2026-08-24T18:10:31.5845551Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-52n7z2mfccvl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:10:35.3841956Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5bfc3306-10af-4dad-9510-c14ce5bf3457\\\",\\r\\n \\\"activityId\\\": \\\"2664cc8d-1c67-4701-a9ad-6ccfbc1df942\\\",\\r\\n \\\"subActivityId\\\": \\\"573c0901-7d69-4aa6-8cc9-171e30466772\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c7f178d3-71a1-44c2-a3cb-331e0e92f5fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5bfc3306-10af-4dad-9510-c14ce5bf3457 ARID=2664cc8d-1c67-4701-a9ad-6ccfbc1df942 \\u003e GW.Http.CallContext/2664cc8d-1c67-4701-a9ad-6ccfbc1df942 \\u003e GW.ExecuteQuery/c7f178d3-71a1-44c2-a3cb-331e0e92f5fb \\u003e DN.FE.ExecuteQuery/573c0901-7d69-4aa6-8cc9-171e30466772)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:10:35.526]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:10:31.5845546Z);\\nlet _endTime = datetime(2026-08-24T18:10:31.5845551Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-52n7z2mfccvl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:10:35.3841956Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5bfc3306-10af-4dad-9510-c14ce5bf3457\\\",\\r\\n \\\"activityId\\\": \\\"2664cc8d-1c67-4701-a9ad-6ccfbc1df942\\\",\\r\\n \\\"subActivityId\\\": \\\"573c0901-7d69-4aa6-8cc9-171e30466772\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c7f178d3-71a1-44c2-a3cb-331e0e92f5fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5bfc3306-10af-4dad-9510-c14ce5bf3457 ARID=2664cc8d-1c67-4701-a9ad-6ccfbc1df942 \\u003e GW.Http.CallContext/2664cc8d-1c67-4701-a9ad-6ccfbc1df942 \\u003e GW.ExecuteQuery/c7f178d3-71a1-44c2-a3cb-331e0e92f5fb \\u003e DN.FE.ExecuteQuery/573c0901-7d69-4aa6-8cc9-171e30466772)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001aa20b0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[18:10:24.297]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:10:20.4586249Z);\\nlet _endTime = datetime(2026-08-24T18:10:20.4586253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-52n7z2mfccvl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:10:24.2714273Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;12b2ecfe-09e1-4838-993b-5dcf60d0b406\\\",\\r\\n \\\"activityId\\\": \\\"04908727-e2e1-425c-ac4a-03f0802e4426\\\",\\r\\n \\\"subActivityId\\\": \\\"c2938410-8cf8-48ad-984d-fb727921dbd7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6ec85876-5223-4015-aeb3-32640b5a0ad9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;12b2ecfe-09e1-4838-993b-5dcf60d0b406 ARID=04908727-e2e1-425c-ac4a-03f0802e4426 \\u003e GW.Http.CallContext/04908727-e2e1-425c-ac4a-03f0802e4426 \\u003e GW.ExecuteQuery/6ec85876-5223-4015-aeb3-32640b5a0ad9 \\u003e DN.FE.ExecuteQuery/c2938410-8cf8-48ad-984d-fb727921dbd7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:10:24.356]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:10:20.4586249Z);\\nlet _endTime = datetime(2026-08-24T18:10:20.4586253Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-52n7z2mfccvl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:10:24.2714273Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;12b2ecfe-09e1-4838-993b-5dcf60d0b406\\\",\\r\\n \\\"activityId\\\": \\\"04908727-e2e1-425c-ac4a-03f0802e4426\\\",\\r\\n \\\"subActivityId\\\": \\\"c2938410-8cf8-48ad-984d-fb727921dbd7\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"6ec85876-5223-4015-aeb3-32640b5a0ad9\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;12b2ecfe-09e1-4838-993b-5dcf60d0b406 ARID=04908727-e2e1-425c-ac4a-03f0802e4426 \\u003e GW.Http.CallContext/04908727-e2e1-425c-ac4a-03f0802e4426 \\u003e GW.ExecuteQuery/6ec85876-5223-4015-aeb3-32640b5a0ad9 \\u003e DN.FE.ExecuteQuery/c2938410-8cf8-48ad-984d-fb727921dbd7)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[18:10:27.782]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:10:24.4180884Z);\\nlet _endTime = datetime(2026-08-24T18:10:24.4180888Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-52n7z2mfccvl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:10:27.7556057Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5c5731ff-386a-4c60-a4ae-7e8099e48b63\\\",\\r\\n \\\"activityId\\\": \\\"ccca268d-b5b0-44b7-bb2c-53f62ce07692\\\",\\r\\n \\\"subActivityId\\\": \\\"929136dd-2380-48b4-9972-3d2d6b49e1e2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"08a8bbb1-cd7f-44af-aa81-2a8d989464b8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5c5731ff-386a-4c60-a4ae-7e8099e48b63 ARID=ccca268d-b5b0-44b7-bb2c-53f62ce07692 \\u003e GW.Http.CallContext/ccca268d-b5b0-44b7-bb2c-53f62ce07692 \\u003e GW.ExecuteQuery/08a8bbb1-cd7f-44af-aa81-2a8d989464b8 \\u003e DN.FE.ExecuteQuery/929136dd-2380-48b4-9972-3d2d6b49e1e2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:10:27.872]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:10:24.4180884Z);\\nlet _endTime = datetime(2026-08-24T18:10:24.4180888Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-52n7z2mfccvl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:10:27.7556057Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5c5731ff-386a-4c60-a4ae-7e8099e48b63\\\",\\r\\n \\\"activityId\\\": \\\"ccca268d-b5b0-44b7-bb2c-53f62ce07692\\\",\\r\\n \\\"subActivityId\\\": \\\"929136dd-2380-48b4-9972-3d2d6b49e1e2\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"08a8bbb1-cd7f-44af-aa81-2a8d989464b8\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5c5731ff-386a-4c60-a4ae-7e8099e48b63 ARID=ccca268d-b5b0-44b7-bb2c-53f62ce07692 \\u003e GW.Http.CallContext/ccca268d-b5b0-44b7-bb2c-53f62ce07692 \\u003e GW.ExecuteQuery/08a8bbb1-cd7f-44af-aa81-2a8d989464b8 \\u003e DN.FE.ExecuteQuery/929136dd-2380-48b4-9972-3d2d6b49e1e2)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[18:10:35.410]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:10:31.5845546Z);\\nlet _endTime = datetime(2026-08-24T18:10:31.5845551Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-52n7z2mfccvl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:10:35.3841956Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5bfc3306-10af-4dad-9510-c14ce5bf3457\\\",\\r\\n \\\"activityId\\\": \\\"2664cc8d-1c67-4701-a9ad-6ccfbc1df942\\\",\\r\\n \\\"subActivityId\\\": \\\"573c0901-7d69-4aa6-8cc9-171e30466772\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c7f178d3-71a1-44c2-a3cb-331e0e92f5fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5bfc3306-10af-4dad-9510-c14ce5bf3457 ARID=2664cc8d-1c67-4701-a9ad-6ccfbc1df942 \\u003e GW.Http.CallContext/2664cc8d-1c67-4701-a9ad-6ccfbc1df942 \\u003e GW.ExecuteQuery/c7f178d3-71a1-44c2-a3cb-331e0e92f5fb \\u003e DN.FE.ExecuteQuery/573c0901-7d69-4aa6-8cc9-171e30466772)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:10:35.526]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:10:31.5845546Z);\\nlet _endTime = datetime(2026-08-24T18:10:31.5845551Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-52n7z2mfccvl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:10:35.3841956Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;5bfc3306-10af-4dad-9510-c14ce5bf3457\\\",\\r\\n \\\"activityId\\\": \\\"2664cc8d-1c67-4701-a9ad-6ccfbc1df942\\\",\\r\\n \\\"subActivityId\\\": \\\"573c0901-7d69-4aa6-8cc9-171e30466772\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c7f178d3-71a1-44c2-a3cb-331e0e92f5fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;5bfc3306-10af-4dad-9510-c14ce5bf3457 ARID=2664cc8d-1c67-4701-a9ad-6ccfbc1df942 \\u003e GW.Http.CallContext/2664cc8d-1c67-4701-a9ad-6ccfbc1df942 \\u003e GW.ExecuteQuery/c7f178d3-71a1-44c2-a3cb-331e0e92f5fb \\u003e DN.FE.ExecuteQuery/573c0901-7d69-4aa6-8cc9-171e30466772)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"7fd872217ad1ddcefa1404adaa1a8f0570c0298b5ceb9cb049ae7786895d4034","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932257060130816","occurred_at":"2026-08-24T16:54:39Z","failed_at":"e2e","signature_id":"70c12268048fb1a990e1fcfe48d8d53530e6791148ebcbe91068d54355e50247","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-552s2ntf2vdp/cluster-candidate-4-20-j2m8wl should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001678380\u003e: \n failed to create HCP cluster cluster-candidate-4-20-j2m8wl: failed waiting for cluster=\"cluster-candidate-4-20-j2m8wl\" in resourcegroup=\"rg-candidate-4-20-552s2ntf2vdp\" to finish creating: GET https://rp.j0130816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2542dd96-3399-4c2c-935f-47bf06d32f0d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2542dd96-3399-4c2c-935f-47bf06d32f0d\",\n \"name\": \"2542dd96-3399-4c2c-935f-47bf06d32f0d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:45:24.52790269Z\",\n \"endTime\": \"2026-08-24T18:04:27.860524772Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-552s2ntf2vdp/cluster-candidate-4-20-j2m8wl should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001678380\u003e: \n failed to create HCP cluster cluster-candidate-4-20-j2m8wl: failed waiting for cluster=\"cluster-candidate-4-20-j2m8wl\" in resourcegroup=\"rg-candidate-4-20-552s2ntf2vdp\" to finish creating: GET https://rp.j0130816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2542dd96-3399-4c2c-935f-47bf06d32f0d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2542dd96-3399-4c2c-935f-47bf06d32f0d\",\n \"name\": \"2542dd96-3399-4c2c-935f-47bf06d32f0d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:45:24.52790269Z\",\n \"endTime\": \"2026-08-24T18:04:27.860524772Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932359686361088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6658,"pr_state":"open","pr_sha":"9906d38df8dc349a74b8507e4059d71747a87617","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T16:54:34Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"16df1ddeaf3af2ec0ffd98b605671967b0a15d80402768ee782c6ac24104f87f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932359686361088","occurred_at":"2026-08-24T16:54:34Z","failed_at":"e2e","signature_id":"5fa5e4ca1b4d0f869a74c660939188cb0fcaacbc16b74da34e727232c29abbff","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 606.493s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00110c300\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[18:17:21.068]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:17:17.9098588Z);\\nlet _endTime = datetime(2026-08-24T18:17:17.9098594Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hw6zxbtrjc8s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:17:21.0429499Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;50177515-800c-48a2-bc9a-6190f8f64647\\\",\\r\\n \\\"activityId\\\": \\\"25584a54-0763-4948-aaed-8fd131374fe3\\\",\\r\\n \\\"subActivityId\\\": \\\"83a66406-6dab-4195-b310-e1df353b33ce\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"18a6bb5a-fff3-4c81-8c7a-669d3ce47a8e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;50177515-800c-48a2-bc9a-6190f8f64647 ARID=25584a54-0763-4948-aaed-8fd131374fe3 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9e4a2718-7c90-4bde-933d-0a3e9ed9c527 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9e4a2718-7c90-4bde-933d-0a3e9ed9c527 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/18a6bb5a-fff3-4c81-8c7a-669d3ce47a8e \\u003e DN.FE.ExecuteQuery/83a66406-6dab-4195-b310-e1df353b33ce)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:17:21.123]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:17:17.9098588Z);\\nlet _endTime = datetime(2026-08-24T18:17:17.9098594Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hw6zxbtrjc8s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:17:21.0429499Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;50177515-800c-48a2-bc9a-6190f8f64647\\\",\\r\\n \\\"activityId\\\": \\\"25584a54-0763-4948-aaed-8fd131374fe3\\\",\\r\\n \\\"subActivityId\\\": \\\"83a66406-6dab-4195-b310-e1df353b33ce\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"18a6bb5a-fff3-4c81-8c7a-669d3ce47a8e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;50177515-800c-48a2-bc9a-6190f8f64647 ARID=25584a54-0763-4948-aaed-8fd131374fe3 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9e4a2718-7c90-4bde-933d-0a3e9ed9c527 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9e4a2718-7c90-4bde-933d-0a3e9ed9c527 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/18a6bb5a-fff3-4c81-8c7a-669d3ce47a8e \\u003e DN.FE.ExecuteQuery/83a66406-6dab-4195-b310-e1df353b33ce)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[18:17:24.187]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:17:21.1899023Z);\\nlet _endTime = datetime(2026-08-24T18:17:21.1899027Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hw6zxbtrjc8s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:17:24.1627418Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;deaa39c3-e50e-4c32-9678-afe919848447\\\",\\r\\n \\\"activityId\\\": \\\"d854a2b3-b483-42b6-b063-920d81b38162\\\",\\r\\n \\\"subActivityId\\\": \\\"92aa21a9-859f-48e1-867d-a294279ddca3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c8f9a286-37c0-4d0a-a38e-876553d01a23\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;deaa39c3-e50e-4c32-9678-afe919848447 ARID=d854a2b3-b483-42b6-b063-920d81b38162 \\u003e GW.Http.CallContext/d854a2b3-b483-42b6-b063-920d81b38162 \\u003e GW.ExecuteQuery/c8f9a286-37c0-4d0a-a38e-876553d01a23 \\u003e DN.FE.ExecuteQuery/92aa21a9-859f-48e1-867d-a294279ddca3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:17:24.238]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:17:21.1899023Z);\\nlet _endTime = datetime(2026-08-24T18:17:21.1899027Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hw6zxbtrjc8s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:17:24.1627418Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;deaa39c3-e50e-4c32-9678-afe919848447\\\",\\r\\n \\\"activityId\\\": \\\"d854a2b3-b483-42b6-b063-920d81b38162\\\",\\r\\n \\\"subActivityId\\\": \\\"92aa21a9-859f-48e1-867d-a294279ddca3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c8f9a286-37c0-4d0a-a38e-876553d01a23\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;deaa39c3-e50e-4c32-9678-afe919848447 ARID=d854a2b3-b483-42b6-b063-920d81b38162 \\u003e GW.Http.CallContext/d854a2b3-b483-42b6-b063-920d81b38162 \\u003e GW.ExecuteQuery/c8f9a286-37c0-4d0a-a38e-876553d01a23 \\u003e DN.FE.ExecuteQuery/92aa21a9-859f-48e1-867d-a294279ddca3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[18:17:30.943]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:17:27.5661103Z);\\nlet _endTime = datetime(2026-08-24T18:17:27.5661107Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hw6zxbtrjc8s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:17:30.9226160Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;fdeb6cbe-caec-42b0-9433-71e30f446206\\\",\\r\\n \\\"activityId\\\": \\\"6b5b443c-b5c8-46e9-a846-0f4ba5b9ed9d\\\",\\r\\n \\\"subActivityId\\\": \\\"fe3732e9-d093-4763-911d-1a8ebf675d35\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2e6c7717-e2dd-4e85-8d2d-85ccd39fd928\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;fdeb6cbe-caec-42b0-9433-71e30f446206 ARID=6b5b443c-b5c8-46e9-a846-0f4ba5b9ed9d \\u003e GW.Http.CallContext/6b5b443c-b5c8-46e9-a846-0f4ba5b9ed9d \\u003e GW.ExecuteQuery/2e6c7717-e2dd-4e85-8d2d-85ccd39fd928 \\u003e DN.FE.ExecuteQuery/fe3732e9-d093-4763-911d-1a8ebf675d35)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:17:31.032]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:17:27.5661103Z);\\nlet _endTime = datetime(2026-08-24T18:17:27.5661107Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hw6zxbtrjc8s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:17:30.9226160Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;fdeb6cbe-caec-42b0-9433-71e30f446206\\\",\\r\\n \\\"activityId\\\": \\\"6b5b443c-b5c8-46e9-a846-0f4ba5b9ed9d\\\",\\r\\n \\\"subActivityId\\\": \\\"fe3732e9-d093-4763-911d-1a8ebf675d35\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2e6c7717-e2dd-4e85-8d2d-85ccd39fd928\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;fdeb6cbe-caec-42b0-9433-71e30f446206 ARID=6b5b443c-b5c8-46e9-a846-0f4ba5b9ed9d \\u003e GW.Http.CallContext/6b5b443c-b5c8-46e9-a846-0f4ba5b9ed9d \\u003e GW.ExecuteQuery/2e6c7717-e2dd-4e85-8d2d-85ccd39fd928 \\u003e DN.FE.ExecuteQuery/fe3732e9-d093-4763-911d-1a8ebf675d35)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 606.493s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00110c300\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[18:17:21.068]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:17:17.9098588Z);\\nlet _endTime = datetime(2026-08-24T18:17:17.9098594Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hw6zxbtrjc8s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:17:21.0429499Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;50177515-800c-48a2-bc9a-6190f8f64647\\\",\\r\\n \\\"activityId\\\": \\\"25584a54-0763-4948-aaed-8fd131374fe3\\\",\\r\\n \\\"subActivityId\\\": \\\"83a66406-6dab-4195-b310-e1df353b33ce\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"18a6bb5a-fff3-4c81-8c7a-669d3ce47a8e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;50177515-800c-48a2-bc9a-6190f8f64647 ARID=25584a54-0763-4948-aaed-8fd131374fe3 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9e4a2718-7c90-4bde-933d-0a3e9ed9c527 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9e4a2718-7c90-4bde-933d-0a3e9ed9c527 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/18a6bb5a-fff3-4c81-8c7a-669d3ce47a8e \\u003e DN.FE.ExecuteQuery/83a66406-6dab-4195-b310-e1df353b33ce)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:17:21.123]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:17:17.9098588Z);\\nlet _endTime = datetime(2026-08-24T18:17:17.9098594Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hw6zxbtrjc8s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:17:21.0429499Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;50177515-800c-48a2-bc9a-6190f8f64647\\\",\\r\\n \\\"activityId\\\": \\\"25584a54-0763-4948-aaed-8fd131374fe3\\\",\\r\\n \\\"subActivityId\\\": \\\"83a66406-6dab-4195-b310-e1df353b33ce\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"18a6bb5a-fff3-4c81-8c7a-669d3ce47a8e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;50177515-800c-48a2-bc9a-6190f8f64647 ARID=25584a54-0763-4948-aaed-8fd131374fe3 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9e4a2718-7c90-4bde-933d-0a3e9ed9c527 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9e4a2718-7c90-4bde-933d-0a3e9ed9c527 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/18a6bb5a-fff3-4c81-8c7a-669d3ce47a8e \\u003e DN.FE.ExecuteQuery/83a66406-6dab-4195-b310-e1df353b33ce)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[18:17:24.187]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:17:21.1899023Z);\\nlet _endTime = datetime(2026-08-24T18:17:21.1899027Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hw6zxbtrjc8s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:17:24.1627418Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;deaa39c3-e50e-4c32-9678-afe919848447\\\",\\r\\n \\\"activityId\\\": \\\"d854a2b3-b483-42b6-b063-920d81b38162\\\",\\r\\n \\\"subActivityId\\\": \\\"92aa21a9-859f-48e1-867d-a294279ddca3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c8f9a286-37c0-4d0a-a38e-876553d01a23\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;deaa39c3-e50e-4c32-9678-afe919848447 ARID=d854a2b3-b483-42b6-b063-920d81b38162 \\u003e GW.Http.CallContext/d854a2b3-b483-42b6-b063-920d81b38162 \\u003e GW.ExecuteQuery/c8f9a286-37c0-4d0a-a38e-876553d01a23 \\u003e DN.FE.ExecuteQuery/92aa21a9-859f-48e1-867d-a294279ddca3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:17:24.238]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:17:21.1899023Z);\\nlet _endTime = datetime(2026-08-24T18:17:21.1899027Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hw6zxbtrjc8s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:17:24.1627418Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;deaa39c3-e50e-4c32-9678-afe919848447\\\",\\r\\n \\\"activityId\\\": \\\"d854a2b3-b483-42b6-b063-920d81b38162\\\",\\r\\n \\\"subActivityId\\\": \\\"92aa21a9-859f-48e1-867d-a294279ddca3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"c8f9a286-37c0-4d0a-a38e-876553d01a23\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;deaa39c3-e50e-4c32-9678-afe919848447 ARID=d854a2b3-b483-42b6-b063-920d81b38162 \\u003e GW.Http.CallContext/d854a2b3-b483-42b6-b063-920d81b38162 \\u003e GW.ExecuteQuery/c8f9a286-37c0-4d0a-a38e-876553d01a23 \\u003e DN.FE.ExecuteQuery/92aa21a9-859f-48e1-867d-a294279ddca3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[18:17:30.943]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:17:27.5661103Z);\\nlet _endTime = datetime(2026-08-24T18:17:27.5661107Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hw6zxbtrjc8s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:17:30.9226160Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;fdeb6cbe-caec-42b0-9433-71e30f446206\\\",\\r\\n \\\"activityId\\\": \\\"6b5b443c-b5c8-46e9-a846-0f4ba5b9ed9d\\\",\\r\\n \\\"subActivityId\\\": \\\"fe3732e9-d093-4763-911d-1a8ebf675d35\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2e6c7717-e2dd-4e85-8d2d-85ccd39fd928\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;fdeb6cbe-caec-42b0-9433-71e30f446206 ARID=6b5b443c-b5c8-46e9-a846-0f4ba5b9ed9d \\u003e GW.Http.CallContext/6b5b443c-b5c8-46e9-a846-0f4ba5b9ed9d \\u003e GW.ExecuteQuery/2e6c7717-e2dd-4e85-8d2d-85ccd39fd928 \\u003e DN.FE.ExecuteQuery/fe3732e9-d093-4763-911d-1a8ebf675d35)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:17:31.032]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:17:27.5661103Z);\\nlet _endTime = datetime(2026-08-24T18:17:27.5661107Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hw6zxbtrjc8s'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:17:30.9226160Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;fdeb6cbe-caec-42b0-9433-71e30f446206\\\",\\r\\n \\\"activityId\\\": \\\"6b5b443c-b5c8-46e9-a846-0f4ba5b9ed9d\\\",\\r\\n \\\"subActivityId\\\": \\\"fe3732e9-d093-4763-911d-1a8ebf675d35\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2e6c7717-e2dd-4e85-8d2d-85ccd39fd928\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;fdeb6cbe-caec-42b0-9433-71e30f446206 ARID=6b5b443c-b5c8-46e9-a846-0f4ba5b9ed9d \\u003e GW.Http.CallContext/6b5b443c-b5c8-46e9-a846-0f4ba5b9ed9d \\u003e GW.ExecuteQuery/2e6c7717-e2dd-4e85-8d2d-85ccd39fd928 \\u003e DN.FE.ExecuteQuery/fe3732e9-d093-4763-911d-1a8ebf675d35)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"67657e8adaefbf1f0963874f0603f93d5b6406a363eca01328d2ab4340544de6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932359686361088","occurred_at":"2026-08-24T16:54:34Z","failed_at":"alert","signature_id":"a3d3dfe16c45487c7de4833dcff7c72d504e34a1b391992ceada6266956233ad","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T17:55:35Z\n Ended: 2026-08-24T18:27:33Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6361088-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6361088-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"54bb2db04ddf012dceee9c7b7444143151cf3825","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T16:43:08Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"1ffe4249fc6a773253a431b7c9c2d9eed0c857fd097e2b2b9cbc46cb0af78525","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","failed_at":"alert","signature_id":"21c20c66c7c9aa8660ddbc1f0e9827cf7f85ba5f2e371e49fb9660d2202ea0a2","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T18:15:21Z\n Ended: 2026-08-24T18:36:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"2db71cb061cdc14b90ac2faadf6f329702213a94f00f7dec1e68078864574c47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","failed_at":"alert","signature_id":"97ee0c31f908f1dbb20c9b6757ed42d4855cc07c6e4c69aed451babbcc2b480c","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T18:01:25Z\n Ended: 2026-08-24T18:34:22Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T18:01:25Z\n Ended: 2026-08-24T18:34:22Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"4dca1a6d6a17d314cf58a7ce0ae5cb9bff9e57fdcd684f0d1eb1b1399485c4ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","failed_at":"e2e","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"8748ee2df0d89ad3a769bb2a4919df0b5953f3fe3e5dc332327391e11e280ab3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","failed_at":"alert","signature_id":"a6a83541f08b4359551ade88cceb74431bde5d979f13cb6852b8c8bc1013fd98","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T18:15:41Z\n Ended: 2026-08-24T18:21:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T18:15:41Z\n Ended: 2026-08-24T18:22:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-24T18:15:41Z\n Ended: 2026-08-24T18:22:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-24T18:15:41Z\n Ended: 2026-08-24T18:23:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-24T18:15:41Z\n Ended: 2026-08-24T18:22:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-24T18:15:41Z\n Ended: 2026-08-24T18:23:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"cb9819a921a4f967fd79ef40857773c045a6ba4b3be665689b035125b39825c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","failed_at":"e2e","signature_id":"6ff77bfa149b82860b0429a7dfe5ab2b71a58982ddc24dc6fbf5996e7d1dca8d","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000ead3d0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[18:00:58.615]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:00:50.5032793Z);\\nlet _endTime = datetime(2026-08-24T18:00:50.5032799Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tpxlr6vxklmr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:00:58.5927495Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;960b7b75-5e41-44c9-8108-4f52ccc0d311\\\",\\r\\n \\\"activityId\\\": \\\"20001fbf-b892-4112-a9ca-caee90ec5f4d\\\",\\r\\n \\\"subActivityId\\\": \\\"b6b4f2e0-1d41-4ffe-bab5-da795807f34f\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"76d1ef04-9778-412c-9b76-1314a76809fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;960b7b75-5e41-44c9-8108-4f52ccc0d311 ARID=20001fbf-b892-4112-a9ca-caee90ec5f4d \\u003e GW.Http.CallContext/20001fbf-b892-4112-a9ca-caee90ec5f4d \\u003e GW.ExecuteQuery/76d1ef04-9778-412c-9b76-1314a76809fb \\u003e DN.FE.ExecuteQuery/b6b4f2e0-1d41-4ffe-bab5-da795807f34f)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:00:58.712]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:00:50.5032793Z);\\nlet _endTime = datetime(2026-08-24T18:00:50.5032799Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tpxlr6vxklmr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:00:58.5927495Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;960b7b75-5e41-44c9-8108-4f52ccc0d311\\\",\\r\\n \\\"activityId\\\": \\\"20001fbf-b892-4112-a9ca-caee90ec5f4d\\\",\\r\\n \\\"subActivityId\\\": \\\"b6b4f2e0-1d41-4ffe-bab5-da795807f34f\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"76d1ef04-9778-412c-9b76-1314a76809fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;960b7b75-5e41-44c9-8108-4f52ccc0d311 ARID=20001fbf-b892-4112-a9ca-caee90ec5f4d \\u003e GW.Http.CallContext/20001fbf-b892-4112-a9ca-caee90ec5f4d \\u003e GW.ExecuteQuery/76d1ef04-9778-412c-9b76-1314a76809fb \\u003e DN.FE.ExecuteQuery/b6b4f2e0-1d41-4ffe-bab5-da795807f34f)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[18:01:06.125]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:00:58.8699941Z);\\nlet _endTime = datetime(2026-08-24T18:00:58.8699948Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tpxlr6vxklmr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:01:06.0919619Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b7b85d2a-f498-4e2b-ba1d-88839731fa27\\\",\\r\\n \\\"activityId\\\": \\\"56a132e1-8a08-4840-b33d-04f01dd6e5d3\\\",\\r\\n \\\"subActivityId\\\": \\\"dec8312a-b74e-46c5-b158-9db7506079c3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"53d73257-f47b-435c-89d6-f90c4b661777\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b7b85d2a-f498-4e2b-ba1d-88839731fa27 ARID=56a132e1-8a08-4840-b33d-04f01dd6e5d3 \\u003e GW.Http.CallContext/56a132e1-8a08-4840-b33d-04f01dd6e5d3 \\u003e GW.ExecuteQuery/53d73257-f47b-435c-89d6-f90c4b661777 \\u003e DN.FE.ExecuteQuery/dec8312a-b74e-46c5-b158-9db7506079c3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:01:06.626]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:00:58.8699941Z);\\nlet _endTime = datetime(2026-08-24T18:00:58.8699948Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tpxlr6vxklmr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:01:06.0919619Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b7b85d2a-f498-4e2b-ba1d-88839731fa27\\\",\\r\\n \\\"activityId\\\": \\\"56a132e1-8a08-4840-b33d-04f01dd6e5d3\\\",\\r\\n \\\"subActivityId\\\": \\\"dec8312a-b74e-46c5-b158-9db7506079c3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"53d73257-f47b-435c-89d6-f90c4b661777\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b7b85d2a-f498-4e2b-ba1d-88839731fa27 ARID=56a132e1-8a08-4840-b33d-04f01dd6e5d3 \\u003e GW.Http.CallContext/56a132e1-8a08-4840-b33d-04f01dd6e5d3 \\u003e GW.ExecuteQuery/53d73257-f47b-435c-89d6-f90c4b661777 \\u003e DN.FE.ExecuteQuery/dec8312a-b74e-46c5-b158-9db7506079c3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[18:01:23.014]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:01:15.4440366Z);\\nlet _endTime = datetime(2026-08-24T18:01:15.4440371Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tpxlr6vxklmr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:01:22.9872186Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;407e8e47-c034-4e41-9520-8819e14f5893\\\",\\r\\n \\\"activityId\\\": \\\"5bdc704e-4e63-48a3-9d02-31d081e3ede4\\\",\\r\\n \\\"subActivityId\\\": \\\"6e8b566e-338b-412a-ade7-cd2f9e654f08\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ff01224f-8ad0-44ce-9ce7-3090f402643e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;407e8e47-c034-4e41-9520-8819e14f5893 ARID=5bdc704e-4e63-48a3-9d02-31d081e3ede4 \\u003e GW.Http.CallContext/5bdc704e-4e63-48a3-9d02-31d081e3ede4 \\u003e GW.ExecuteQuery/ff01224f-8ad0-44ce-9ce7-3090f402643e \\u003e DN.FE.ExecuteQuery/6e8b566e-338b-412a-ade7-cd2f9e654f08)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:01:23.290]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:01:15.4440366Z);\\nlet _endTime = datetime(2026-08-24T18:01:15.4440371Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tpxlr6vxklmr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:01:22.9872186Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;407e8e47-c034-4e41-9520-8819e14f5893\\\",\\r\\n \\\"activityId\\\": \\\"5bdc704e-4e63-48a3-9d02-31d081e3ede4\\\",\\r\\n \\\"subActivityId\\\": \\\"6e8b566e-338b-412a-ade7-cd2f9e654f08\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ff01224f-8ad0-44ce-9ce7-3090f402643e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;407e8e47-c034-4e41-9520-8819e14f5893 ARID=5bdc704e-4e63-48a3-9d02-31d081e3ede4 \\u003e GW.Http.CallContext/5bdc704e-4e63-48a3-9d02-31d081e3ede4 \\u003e GW.ExecuteQuery/ff01224f-8ad0-44ce-9ce7-3090f402643e \\u003e DN.FE.ExecuteQuery/6e8b566e-338b-412a-ade7-cd2f9e654f08)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000ead3d0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[18:00:58.615]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:00:50.5032793Z);\\nlet _endTime = datetime(2026-08-24T18:00:50.5032799Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tpxlr6vxklmr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:00:58.5927495Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;960b7b75-5e41-44c9-8108-4f52ccc0d311\\\",\\r\\n \\\"activityId\\\": \\\"20001fbf-b892-4112-a9ca-caee90ec5f4d\\\",\\r\\n \\\"subActivityId\\\": \\\"b6b4f2e0-1d41-4ffe-bab5-da795807f34f\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"76d1ef04-9778-412c-9b76-1314a76809fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;960b7b75-5e41-44c9-8108-4f52ccc0d311 ARID=20001fbf-b892-4112-a9ca-caee90ec5f4d \\u003e GW.Http.CallContext/20001fbf-b892-4112-a9ca-caee90ec5f4d \\u003e GW.ExecuteQuery/76d1ef04-9778-412c-9b76-1314a76809fb \\u003e DN.FE.ExecuteQuery/b6b4f2e0-1d41-4ffe-bab5-da795807f34f)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:00:58.712]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:00:50.5032793Z);\\nlet _endTime = datetime(2026-08-24T18:00:50.5032799Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tpxlr6vxklmr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:00:58.5927495Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;960b7b75-5e41-44c9-8108-4f52ccc0d311\\\",\\r\\n \\\"activityId\\\": \\\"20001fbf-b892-4112-a9ca-caee90ec5f4d\\\",\\r\\n \\\"subActivityId\\\": \\\"b6b4f2e0-1d41-4ffe-bab5-da795807f34f\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"76d1ef04-9778-412c-9b76-1314a76809fb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;960b7b75-5e41-44c9-8108-4f52ccc0d311 ARID=20001fbf-b892-4112-a9ca-caee90ec5f4d \\u003e GW.Http.CallContext/20001fbf-b892-4112-a9ca-caee90ec5f4d \\u003e GW.ExecuteQuery/76d1ef04-9778-412c-9b76-1314a76809fb \\u003e DN.FE.ExecuteQuery/b6b4f2e0-1d41-4ffe-bab5-da795807f34f)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[18:01:06.125]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:00:58.8699941Z);\\nlet _endTime = datetime(2026-08-24T18:00:58.8699948Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tpxlr6vxklmr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:01:06.0919619Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b7b85d2a-f498-4e2b-ba1d-88839731fa27\\\",\\r\\n \\\"activityId\\\": \\\"56a132e1-8a08-4840-b33d-04f01dd6e5d3\\\",\\r\\n \\\"subActivityId\\\": \\\"dec8312a-b74e-46c5-b158-9db7506079c3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"53d73257-f47b-435c-89d6-f90c4b661777\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b7b85d2a-f498-4e2b-ba1d-88839731fa27 ARID=56a132e1-8a08-4840-b33d-04f01dd6e5d3 \\u003e GW.Http.CallContext/56a132e1-8a08-4840-b33d-04f01dd6e5d3 \\u003e GW.ExecuteQuery/53d73257-f47b-435c-89d6-f90c4b661777 \\u003e DN.FE.ExecuteQuery/dec8312a-b74e-46c5-b158-9db7506079c3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:01:06.626]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:00:58.8699941Z);\\nlet _endTime = datetime(2026-08-24T18:00:58.8699948Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tpxlr6vxklmr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:01:06.0919619Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;b7b85d2a-f498-4e2b-ba1d-88839731fa27\\\",\\r\\n \\\"activityId\\\": \\\"56a132e1-8a08-4840-b33d-04f01dd6e5d3\\\",\\r\\n \\\"subActivityId\\\": \\\"dec8312a-b74e-46c5-b158-9db7506079c3\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"53d73257-f47b-435c-89d6-f90c4b661777\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;b7b85d2a-f498-4e2b-ba1d-88839731fa27 ARID=56a132e1-8a08-4840-b33d-04f01dd6e5d3 \\u003e GW.Http.CallContext/56a132e1-8a08-4840-b33d-04f01dd6e5d3 \\u003e GW.ExecuteQuery/53d73257-f47b-435c-89d6-f90c4b661777 \\u003e DN.FE.ExecuteQuery/dec8312a-b74e-46c5-b158-9db7506079c3)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[18:01:23.014]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:01:15.4440366Z);\\nlet _endTime = datetime(2026-08-24T18:01:15.4440371Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tpxlr6vxklmr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:01:22.9872186Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;407e8e47-c034-4e41-9520-8819e14f5893\\\",\\r\\n \\\"activityId\\\": \\\"5bdc704e-4e63-48a3-9d02-31d081e3ede4\\\",\\r\\n \\\"subActivityId\\\": \\\"6e8b566e-338b-412a-ade7-cd2f9e654f08\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ff01224f-8ad0-44ce-9ce7-3090f402643e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;407e8e47-c034-4e41-9520-8819e14f5893 ARID=5bdc704e-4e63-48a3-9d02-31d081e3ede4 \\u003e GW.Http.CallContext/5bdc704e-4e63-48a3-9d02-31d081e3ede4 \\u003e GW.ExecuteQuery/ff01224f-8ad0-44ce-9ce7-3090f402643e \\u003e DN.FE.ExecuteQuery/6e8b566e-338b-412a-ade7-cd2f9e654f08)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:01:23.290]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:01:15.4440366Z);\\nlet _endTime = datetime(2026-08-24T18:01:15.4440371Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tpxlr6vxklmr'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:01:22.9872186Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;407e8e47-c034-4e41-9520-8819e14f5893\\\",\\r\\n \\\"activityId\\\": \\\"5bdc704e-4e63-48a3-9d02-31d081e3ede4\\\",\\r\\n \\\"subActivityId\\\": \\\"6e8b566e-338b-412a-ade7-cd2f9e654f08\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ff01224f-8ad0-44ce-9ce7-3090f402643e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;407e8e47-c034-4e41-9520-8819e14f5893 ARID=5bdc704e-4e63-48a3-9d02-31d081e3ede4 \\u003e GW.Http.CallContext/5bdc704e-4e63-48a3-9d02-31d081e3ede4 \\u003e GW.ExecuteQuery/ff01224f-8ad0-44ce-9ce7-3090f402643e \\u003e DN.FE.ExecuteQuery/6e8b566e-338b-412a-ade7-cd2f9e654f08)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"e3e7ce0867add699a5c6a1a130d93ffc8b51e80a4934c4f844890a09afdd4e16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","failed_at":"alert","signature_id":"9d5e8ec654bc2ebd399a25878bada85329c8658827f68d63b0042fb7498d8047","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T18:02:20Z\n Ended: 2026-08-24T18:38:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4cebd923-31c4-4c0a-b568-63bf1a281b9a\"\n Description: Pod ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-08-24T18:03:20Z\n Ended: 2026-08-24T18:38:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4cebd923-31c4-4c0a-b568-63bf1a281b9a\"\n Description: Pod ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6591,"pr_state":"open","pr_sha":"de4526f28579c2092fa294ea59d691384b8c84b6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T16:40:39Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"21267f0156ec3ceb26be9cc719ac9e9ff3a8b3e7ebf40e90b5ffaa55ed1a1373","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","failed_at":"e2e","signature_id":"d1005ca6dc326e3c744c86ae1e9fb1086de2e12e4a6351183145527b31e5eccf","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019d4200\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-l2vfsmsdwlz6\" to finish creating: GET https://rp.j0629888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7fff289f-807c-4680-8e8c-4687765fb3f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7fff289f-807c-4680-8e8c-4687765fb3f9\",\n \"name\": \"7fff289f-807c-4680-8e8c-4687765fb3f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:34:15.71678435Z\",\n \"endTime\": \"2026-08-24T17:53:23.588048645Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019d4200\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-l2vfsmsdwlz6\" to finish creating: GET https://rp.j0629888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7fff289f-807c-4680-8e8c-4687765fb3f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7fff289f-807c-4680-8e8c-4687765fb3f9\",\n \"name\": \"7fff289f-807c-4680-8e8c-4687765fb3f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:34:15.71678435Z\",\n \"endTime\": \"2026-08-24T17:53:23.588048645Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"271c6f2e589f88ca5c067157a00404c6d11bcc3b3a57ac601e126733db3c56d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","failed_at":"alert","signature_id":"3eebb3220fdafc37e59ae5d05cda26d2e052d0d1681fedb42f8fa69446913217","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T17:38:07Z\n Ended: 2026-08-24T17:45:05Z\n Severity: Sev3\n Labels: alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j0629888-svc\", component=\"fleet\", name=\"clustersserviceregistrationcontroller\", severity=\"warning\"\n Description: Fleet controller workqueue clustersserviceregistrationcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","failure_pattern":"alert [svc] FleetControllerRetryHotLoop fired","search_query":"alert [svc] FleetControllerRetryHotLoop fired"}},{"row_id":"3e0733a17588b76381c4fe36ca6a9c59192379502884fae9dc0e881d8559897c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","failed_at":"e2e","signature_id":"9806f10f5f28de5b459cf7d30f0fa456f7ab916323bce3ad88103d4bb5b0c102","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-nzmkwtbzb8xk/cluster-candidate-4-21-k47dlw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011d300\u003e: \n failed to create HCP cluster cluster-candidate-4-21-k47dlw: failed waiting for cluster=\"cluster-candidate-4-21-k47dlw\" in resourcegroup=\"rg-candidate-4-21-nzmkwtbzb8xk\" to finish creating: GET https://rp.j0629888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e224492a-2630-4a37-b5e0-7ef00f6cd9dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e224492a-2630-4a37-b5e0-7ef00f6cd9dd\",\n \"name\": \"e224492a-2630-4a37-b5e0-7ef00f6cd9dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:34:09.560283017Z\",\n \"endTime\": \"2026-08-24T17:53:13.775593008Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-nzmkwtbzb8xk/cluster-candidate-4-21-k47dlw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011d300\u003e: \n failed to create HCP cluster cluster-candidate-4-21-k47dlw: failed waiting for cluster=\"cluster-candidate-4-21-k47dlw\" in resourcegroup=\"rg-candidate-4-21-nzmkwtbzb8xk\" to finish creating: GET https://rp.j0629888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e224492a-2630-4a37-b5e0-7ef00f6cd9dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e224492a-2630-4a37-b5e0-7ef00f6cd9dd\",\n \"name\": \"e224492a-2630-4a37-b5e0-7ef00f6cd9dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:34:09.560283017Z\",\n \"endTime\": \"2026-08-24T17:53:13.775593008Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b645ca11afad5b79ec9dae25eb364818cdce5c3fe97c8a6de8581921ac37e5b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4f4c2fb4babbdf0ec41a17859713cfb11518f890058dd2b1a9b35c584cfa14c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","failed_at":"e2e","signature_id":"1ae3c1ffd864572007c734910a8d27ab7b2bea356002ed663c85c3e98b525248","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 602.326s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00174c0b0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[18:02:15.191]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:02:09.5899113Z);\\nlet _endTime = datetime(2026-08-24T18:02:09.5899117Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hxrz24kgz22q'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:02:15.1649816Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;dc1be810-20f7-4576-9c23-8c49c10db75e\\\",\\r\\n \\\"activityId\\\": \\\"8b50e128-6bf1-4c67-b4e1-a93617248240\\\",\\r\\n \\\"subActivityId\\\": \\\"d4d65d0b-3d08-4bae-8e97-02e2aa50a413\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d1dc26ed-fe0d-4b0a-a356-ac3586800c00\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;dc1be810-20f7-4576-9c23-8c49c10db75e ARID=8b50e128-6bf1-4c67-b4e1-a93617248240 \\u003e GW.Http.CallContext/8b50e128-6bf1-4c67-b4e1-a93617248240 \\u003e GW.ExecuteQuery/d1dc26ed-fe0d-4b0a-a356-ac3586800c00 \\u003e DN.FE.ExecuteQuery/d4d65d0b-3d08-4bae-8e97-02e2aa50a413)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:02:15.411]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:02:09.5899113Z);\\nlet _endTime = datetime(2026-08-24T18:02:09.5899117Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hxrz24kgz22q'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:02:15.1649816Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;dc1be810-20f7-4576-9c23-8c49c10db75e\\\",\\r\\n \\\"activityId\\\": \\\"8b50e128-6bf1-4c67-b4e1-a93617248240\\\",\\r\\n \\\"subActivityId\\\": \\\"d4d65d0b-3d08-4bae-8e97-02e2aa50a413\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d1dc26ed-fe0d-4b0a-a356-ac3586800c00\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;dc1be810-20f7-4576-9c23-8c49c10db75e ARID=8b50e128-6bf1-4c67-b4e1-a93617248240 \\u003e GW.Http.CallContext/8b50e128-6bf1-4c67-b4e1-a93617248240 \\u003e GW.ExecuteQuery/d1dc26ed-fe0d-4b0a-a356-ac3586800c00 \\u003e DN.FE.ExecuteQuery/d4d65d0b-3d08-4bae-8e97-02e2aa50a413)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[18:02:22.018]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:02:15.4920397Z);\\nlet _endTime = datetime(2026-08-24T18:02:15.4920402Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hxrz24kgz22q'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:02:21.9924804Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0d1d7012-9bec-42ca-94eb-41a4c8ac0207\\\",\\r\\n \\\"activityId\\\": \\\"7c88679c-59d7-49a4-a8e0-97888f2c387f\\\",\\r\\n \\\"subActivityId\\\": \\\"20c41293-ee06-43b0-9f85-1faf9b5ab846\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0bf6b207-9c8c-43f0-a099-76d33cc8d11d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0d1d7012-9bec-42ca-94eb-41a4c8ac0207 ARID=7c88679c-59d7-49a4-a8e0-97888f2c387f \\u003e GW.Http.CallContext/7c88679c-59d7-49a4-a8e0-97888f2c387f \\u003e GW.ExecuteQuery/0bf6b207-9c8c-43f0-a099-76d33cc8d11d \\u003e DN.FE.ExecuteQuery/20c41293-ee06-43b0-9f85-1faf9b5ab846)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:02:22.174]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:02:15.4920397Z);\\nlet _endTime = datetime(2026-08-24T18:02:15.4920402Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hxrz24kgz22q'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:02:21.9924804Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0d1d7012-9bec-42ca-94eb-41a4c8ac0207\\\",\\r\\n \\\"activityId\\\": \\\"7c88679c-59d7-49a4-a8e0-97888f2c387f\\\",\\r\\n \\\"subActivityId\\\": \\\"20c41293-ee06-43b0-9f85-1faf9b5ab846\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0bf6b207-9c8c-43f0-a099-76d33cc8d11d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0d1d7012-9bec-42ca-94eb-41a4c8ac0207 ARID=7c88679c-59d7-49a4-a8e0-97888f2c387f \\u003e GW.Http.CallContext/7c88679c-59d7-49a4-a8e0-97888f2c387f \\u003e GW.ExecuteQuery/0bf6b207-9c8c-43f0-a099-76d33cc8d11d \\u003e DN.FE.ExecuteQuery/20c41293-ee06-43b0-9f85-1faf9b5ab846)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[18:02:34.313]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:02:27.8203571Z);\\nlet _endTime = datetime(2026-08-24T18:02:27.8203574Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hxrz24kgz22q'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:02:34.2861093Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;7b69ab3c-142e-42fc-ba7f-d284b2e71d38\\\",\\r\\n \\\"activityId\\\": \\\"833f4c61-46f8-4fdc-895d-d0121d895d64\\\",\\r\\n \\\"subActivityId\\\": \\\"18d5bb74-4c1c-4003-b6ff-d5c3cbb43607\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"cbd0d9b4-40b0-49a2-b3c0-ff7e4292d28e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7b69ab3c-142e-42fc-ba7f-d284b2e71d38 ARID=833f4c61-46f8-4fdc-895d-d0121d895d64 \\u003e GW.Http.CallContext/833f4c61-46f8-4fdc-895d-d0121d895d64 \\u003e GW.ExecuteQuery/cbd0d9b4-40b0-49a2-b3c0-ff7e4292d28e \\u003e DN.FE.ExecuteQuery/18d5bb74-4c1c-4003-b6ff-d5c3cbb43607)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:02:34.447]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:02:27.8203571Z);\\nlet _endTime = datetime(2026-08-24T18:02:27.8203574Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hxrz24kgz22q'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:02:34.2861093Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;7b69ab3c-142e-42fc-ba7f-d284b2e71d38\\\",\\r\\n \\\"activityId\\\": \\\"833f4c61-46f8-4fdc-895d-d0121d895d64\\\",\\r\\n \\\"subActivityId\\\": \\\"18d5bb74-4c1c-4003-b6ff-d5c3cbb43607\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"cbd0d9b4-40b0-49a2-b3c0-ff7e4292d28e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7b69ab3c-142e-42fc-ba7f-d284b2e71d38 ARID=833f4c61-46f8-4fdc-895d-d0121d895d64 \\u003e GW.Http.CallContext/833f4c61-46f8-4fdc-895d-d0121d895d64 \\u003e GW.ExecuteQuery/cbd0d9b4-40b0-49a2-b3c0-ff7e4292d28e \\u003e DN.FE.ExecuteQuery/18d5bb74-4c1c-4003-b6ff-d5c3cbb43607)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 602.326s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00174c0b0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[18:02:15.191]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:02:09.5899113Z);\\nlet _endTime = datetime(2026-08-24T18:02:09.5899117Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hxrz24kgz22q'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:02:15.1649816Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;dc1be810-20f7-4576-9c23-8c49c10db75e\\\",\\r\\n \\\"activityId\\\": \\\"8b50e128-6bf1-4c67-b4e1-a93617248240\\\",\\r\\n \\\"subActivityId\\\": \\\"d4d65d0b-3d08-4bae-8e97-02e2aa50a413\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d1dc26ed-fe0d-4b0a-a356-ac3586800c00\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;dc1be810-20f7-4576-9c23-8c49c10db75e ARID=8b50e128-6bf1-4c67-b4e1-a93617248240 \\u003e GW.Http.CallContext/8b50e128-6bf1-4c67-b4e1-a93617248240 \\u003e GW.ExecuteQuery/d1dc26ed-fe0d-4b0a-a356-ac3586800c00 \\u003e DN.FE.ExecuteQuery/d4d65d0b-3d08-4bae-8e97-02e2aa50a413)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:02:15.411]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:02:09.5899113Z);\\nlet _endTime = datetime(2026-08-24T18:02:09.5899117Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hxrz24kgz22q'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:02:15.1649816Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;dc1be810-20f7-4576-9c23-8c49c10db75e\\\",\\r\\n \\\"activityId\\\": \\\"8b50e128-6bf1-4c67-b4e1-a93617248240\\\",\\r\\n \\\"subActivityId\\\": \\\"d4d65d0b-3d08-4bae-8e97-02e2aa50a413\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d1dc26ed-fe0d-4b0a-a356-ac3586800c00\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;dc1be810-20f7-4576-9c23-8c49c10db75e ARID=8b50e128-6bf1-4c67-b4e1-a93617248240 \\u003e GW.Http.CallContext/8b50e128-6bf1-4c67-b4e1-a93617248240 \\u003e GW.ExecuteQuery/d1dc26ed-fe0d-4b0a-a356-ac3586800c00 \\u003e DN.FE.ExecuteQuery/d4d65d0b-3d08-4bae-8e97-02e2aa50a413)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[18:02:22.018]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:02:15.4920397Z);\\nlet _endTime = datetime(2026-08-24T18:02:15.4920402Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hxrz24kgz22q'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:02:21.9924804Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0d1d7012-9bec-42ca-94eb-41a4c8ac0207\\\",\\r\\n \\\"activityId\\\": \\\"7c88679c-59d7-49a4-a8e0-97888f2c387f\\\",\\r\\n \\\"subActivityId\\\": \\\"20c41293-ee06-43b0-9f85-1faf9b5ab846\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0bf6b207-9c8c-43f0-a099-76d33cc8d11d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0d1d7012-9bec-42ca-94eb-41a4c8ac0207 ARID=7c88679c-59d7-49a4-a8e0-97888f2c387f \\u003e GW.Http.CallContext/7c88679c-59d7-49a4-a8e0-97888f2c387f \\u003e GW.ExecuteQuery/0bf6b207-9c8c-43f0-a099-76d33cc8d11d \\u003e DN.FE.ExecuteQuery/20c41293-ee06-43b0-9f85-1faf9b5ab846)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:02:22.174]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:02:15.4920397Z);\\nlet _endTime = datetime(2026-08-24T18:02:15.4920402Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hxrz24kgz22q'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:02:21.9924804Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;0d1d7012-9bec-42ca-94eb-41a4c8ac0207\\\",\\r\\n \\\"activityId\\\": \\\"7c88679c-59d7-49a4-a8e0-97888f2c387f\\\",\\r\\n \\\"subActivityId\\\": \\\"20c41293-ee06-43b0-9f85-1faf9b5ab846\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"0bf6b207-9c8c-43f0-a099-76d33cc8d11d\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;0d1d7012-9bec-42ca-94eb-41a4c8ac0207 ARID=7c88679c-59d7-49a4-a8e0-97888f2c387f \\u003e GW.Http.CallContext/7c88679c-59d7-49a4-a8e0-97888f2c387f \\u003e GW.ExecuteQuery/0bf6b207-9c8c-43f0-a099-76d33cc8d11d \\u003e DN.FE.ExecuteQuery/20c41293-ee06-43b0-9f85-1faf9b5ab846)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[18:02:34.313]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:02:27.8203571Z);\\nlet _endTime = datetime(2026-08-24T18:02:27.8203574Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hxrz24kgz22q'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:02:34.2861093Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;7b69ab3c-142e-42fc-ba7f-d284b2e71d38\\\",\\r\\n \\\"activityId\\\": \\\"833f4c61-46f8-4fdc-895d-d0121d895d64\\\",\\r\\n \\\"subActivityId\\\": \\\"18d5bb74-4c1c-4003-b6ff-d5c3cbb43607\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"cbd0d9b4-40b0-49a2-b3c0-ff7e4292d28e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7b69ab3c-142e-42fc-ba7f-d284b2e71d38 ARID=833f4c61-46f8-4fdc-895d-d0121d895d64 \\u003e GW.Http.CallContext/833f4c61-46f8-4fdc-895d-d0121d895d64 \\u003e GW.ExecuteQuery/cbd0d9b4-40b0-49a2-b3c0-ff7e4292d28e \\u003e DN.FE.ExecuteQuery/18d5bb74-4c1c-4003-b6ff-d5c3cbb43607)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[18:02:34.447]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T18:02:27.8203571Z);\\nlet _endTime = datetime(2026-08-24T18:02:27.8203574Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-hxrz24kgz22q'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T18:02:34.2861093Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;7b69ab3c-142e-42fc-ba7f-d284b2e71d38\\\",\\r\\n \\\"activityId\\\": \\\"833f4c61-46f8-4fdc-895d-d0121d895d64\\\",\\r\\n \\\"subActivityId\\\": \\\"18d5bb74-4c1c-4003-b6ff-d5c3cbb43607\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"cbd0d9b4-40b0-49a2-b3c0-ff7e4292d28e\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7b69ab3c-142e-42fc-ba7f-d284b2e71d38 ARID=833f4c61-46f8-4fdc-895d-d0121d895d64 \\u003e GW.Http.CallContext/833f4c61-46f8-4fdc-895d-d0121d895d64 \\u003e GW.ExecuteQuery/cbd0d9b4-40b0-49a2-b3c0-ff7e4292d28e \\u003e DN.FE.ExecuteQuery/18d5bb74-4c1c-4003-b6ff-d5c3cbb43607)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"a3d7b4f85943055dec3b364fd02c660620e1d0ed0b30344f24facbd2626bdcd4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","failed_at":"e2e","signature_id":"688dcfad091ac3bb942f27fcd5e2e3d6471f49c9042338c1c965ab6c339ae7e6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-hbs92nmx4z9w/cluster-candidate-5-0-v6w5lz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a025e0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-v6w5lz: failed waiting for cluster=\"cluster-candidate-5-0-v6w5lz\" in resourcegroup=\"rg-candidate-5-0-hbs92nmx4z9w\" to finish creating: GET https://rp.j0629888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1af9d894-18ff-4381-9686-ac8da4e5e05a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1af9d894-18ff-4381-9686-ac8da4e5e05a\",\n \"name\": \"1af9d894-18ff-4381-9686-ac8da4e5e05a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:34:08.691049831Z\",\n \"endTime\": \"2026-08-24T17:53:13.772213433Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-hbs92nmx4z9w/cluster-candidate-5-0-v6w5lz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a025e0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-v6w5lz: failed waiting for cluster=\"cluster-candidate-5-0-v6w5lz\" in resourcegroup=\"rg-candidate-5-0-hbs92nmx4z9w\" to finish creating: GET https://rp.j0629888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1af9d894-18ff-4381-9686-ac8da4e5e05a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1af9d894-18ff-4381-9686-ac8da4e5e05a\",\n \"name\": \"1af9d894-18ff-4381-9686-ac8da4e5e05a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:34:08.691049831Z\",\n \"endTime\": \"2026-08-24T17:53:13.772213433Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b981eff098f74d9b9a874ceb3aabf6d700ffb9c681f83d88903dfc4a8a21aaa6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","failed_at":"e2e","signature_id":"db07fb00cb7c1dc1d49041e7a151eab39b4fee2bc9901dd00a1fc182625826d2","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009626a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2qrhl2np7wfb\" to finish creating: GET https://rp.j0629888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffc54622-63ff-4342-a7a6-0d895601d00c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffc54622-63ff-4342-a7a6-0d895601d00c\",\n \"name\": \"ffc54622-63ff-4342-a7a6-0d895601d00c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:35:49.514492475Z\",\n \"endTime\": \"2026-08-24T17:54:53.594926153Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009626a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2qrhl2np7wfb\" to finish creating: GET https://rp.j0629888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffc54622-63ff-4342-a7a6-0d895601d00c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffc54622-63ff-4342-a7a6-0d895601d00c\",\n \"name\": \"ffc54622-63ff-4342-a7a6-0d895601d00c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:35:49.514492475Z\",\n \"endTime\": \"2026-08-24T17:54:53.594926153Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bd49d197fe4f2afebb16803d166bb606ca0f8e6bfb9384dab224591e2dc0d9d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","failed_at":"alert","signature_id":"99e8b0c7fce35f618653d3b9144ebffd6644e1f77bf1ee7a7e41e85ca3b0c8eb","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T17:55:40Z\n Ended: 2026-08-24T18:14:40Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0629888-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.209:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0629888/providers/microsoft.monitor/accounts/hcps-j0629888\", namespace=\"ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-tktc6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T17:55:40Z\n Ended: 2026-08-24T18:15:41Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0629888-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.209:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0629888/providers/microsoft.monitor/accounts/hcps-j0629888\", namespace=\"ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-tktc6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"f1becd6fe5b4d681512c63be2e9a64f59126527ce8ed74ec382485b1b0e0964e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","failed_at":"e2e","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","4b645ca11afad5b79ec9dae25eb364818cdce5c3fe97c8a6de8581921ac37e5b","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6657,"pr_state":"open","pr_sha":"716e07da11c908e7c646355ce5a41558b047f150","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T16:34:51Z"},"failed_at":["alert","e2e"],"failed_test_count":13,"occurrences":[{"row_id":"1841a5a1a4292bbc6f2fb89c36d8833c8f1369539b6dd2d73b67ff4188a83637","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"e2e","signature_id":"51d9d625d26042275bccf29b990ed16b6ab0c82bab381da48ae1943a65da776a","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001068580\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ks59qpfw2b7j\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/764346c6-fb81-4c96-9ce9-57d5acf0bfef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/764346c6-fb81-4c96-9ce9-57d5acf0bfef\",\n \"name\": \"764346c6-fb81-4c96-9ce9-57d5acf0bfef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:32.803601268Z\",\n \"endTime\": \"2026-08-24T17:47:34.550152556Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001068580\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ks59qpfw2b7j\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/764346c6-fb81-4c96-9ce9-57d5acf0bfef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/764346c6-fb81-4c96-9ce9-57d5acf0bfef\",\n \"name\": \"764346c6-fb81-4c96-9ce9-57d5acf0bfef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:32.803601268Z\",\n \"endTime\": \"2026-08-24T17:47:34.550152556Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1d3cbacc4de9b326dcf09fa02d7f25001ec45b4c8dfa23a4bba0e8e756286356","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"e2e","signature_id":"94e14a4948dabf46772c6e049c6c316329369f07279d585a70084b3a5b07d8b8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-cbhfzqtvb9th/cluster-candidate-4-21-5bbfxw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea2680\u003e: \n failed to create HCP cluster cluster-candidate-4-21-5bbfxw: failed waiting for cluster=\"cluster-candidate-4-21-5bbfxw\" in resourcegroup=\"rg-candidate-4-21-cbhfzqtvb9th\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n \"name\": \"a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:32.844651935Z\",\n \"endTime\": \"2026-08-24T17:47:34.560561952Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-cbhfzqtvb9th/cluster-candidate-4-21-5bbfxw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea2680\u003e: \n failed to create HCP cluster cluster-candidate-4-21-5bbfxw: failed waiting for cluster=\"cluster-candidate-4-21-5bbfxw\" in resourcegroup=\"rg-candidate-4-21-cbhfzqtvb9th\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n \"name\": \"a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:32.844651935Z\",\n \"endTime\": \"2026-08-24T17:47:34.560561952Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"554b1293a3005dd3cc0a7e8d23d695e50057c353dbb85d067bebad7264fc99d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"e2e","signature_id":"8d53ed6b1258502dfa6cecd4e6049f72772418b7686ae5c9e29c6adc65e73c79","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f64d80\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-5665zqlrt778\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/049b9660-e1f7-4a39-9064-73a72d6b9292\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/049b9660-e1f7-4a39-9064-73a72d6b9292\",\n \"name\": \"049b9660-e1f7-4a39-9064-73a72d6b9292\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:27:52.106192297Z\",\n \"endTime\": \"2026-08-24T17:46:54.569578114Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f64d80\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-5665zqlrt778\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/049b9660-e1f7-4a39-9064-73a72d6b9292\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/049b9660-e1f7-4a39-9064-73a72d6b9292\",\n \"name\": \"049b9660-e1f7-4a39-9064-73a72d6b9292\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:27:52.106192297Z\",\n \"endTime\": \"2026-08-24T17:46:54.569578114Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6f0519b429a79da15994996c4cc69416a143c1f9f90aaf495fa785263c56411c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"alert","signature_id":"ddb17f85d08dfa0f0d60af8e146b060c3f080bb3cb873c1fd946241da0c96c74","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 21 time(s) (1 unknown, 20 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T17:41:05Z\n Ended: 2026-08-24T18:03:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2463616-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd32h2d1lquds92aqdmsnijjli4i6pt-f4l8d4z1z4r3u1t\", pod=\"router-7685ff5cfb-4r4wg\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd32h2d1lquds92aqdmsnijjli4i6pt-f4l8d4z1z4r3u1t/router-7685ff5cfb-4r4wg has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"76b63db2cdf34c564c167bc435fe86a58e348c69fa3088c5e48c0e707250ea17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"e2e","signature_id":"b261dd52f676523d8a989fb0761bdfb6871af15b6d8ed758e7d327e30440bdf8","test_name":"Fleet should have scheduling data for ready management clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:155 with:\nstamp 2 scheduling must have requested resources\nExpected\n \u003cv1.ResourceList | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:155 with:\nstamp 2 scheduling must have requested resources\nExpected\n \u003cv1.ResourceList | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bff39f67862ef72eccdd3c2006d48756f417856635339107ace9fde9b0a679a4","failure_pattern":"stamp 2 scheduling must have requested resources","search_query":"stamp 2 scheduling must have requested resources"}},{"row_id":"805066d79d15fa8c39059a4a349e552a4a39fd1f8baba7983305af82849d258c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"e2e","signature_id":"40b147b3a09185f7fbbd5416adf8d0e158c67796d5387c2aa465c0a43fd1dc30","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-h6q45zq2k92c/cluster-candidate-4-22-ppq8j9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001250700\u003e: \n failed to create HCP cluster cluster-candidate-4-22-ppq8j9: failed waiting for cluster=\"cluster-candidate-4-22-ppq8j9\" in resourcegroup=\"rg-candidate-4-22-h6q45zq2k92c\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ceece4bb-0461-4918-a3ae-9721cd29423d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ceece4bb-0461-4918-a3ae-9721cd29423d\",\n \"name\": \"ceece4bb-0461-4918-a3ae-9721cd29423d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:32.643827373Z\",\n \"endTime\": \"2026-08-24T17:47:34.518536819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-h6q45zq2k92c/cluster-candidate-4-22-ppq8j9 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001250700\u003e: \n failed to create HCP cluster cluster-candidate-4-22-ppq8j9: failed waiting for cluster=\"cluster-candidate-4-22-ppq8j9\" in resourcegroup=\"rg-candidate-4-22-h6q45zq2k92c\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ceece4bb-0461-4918-a3ae-9721cd29423d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ceece4bb-0461-4918-a3ae-9721cd29423d\",\n \"name\": \"ceece4bb-0461-4918-a3ae-9721cd29423d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:32.643827373Z\",\n \"endTime\": \"2026-08-24T17:47:34.518536819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8818c98df621b77cf569cf549b06d049536fd63e53c6cb3ea1893514b9d52d0b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"e2e","signature_id":"2cae48dc9a7ac77e5a8ac54a32e19e04d69dc07b4bbfdffac675fbb83525b59a","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--c84xq6vfx9vp/cluster-hs-pre-drcr6j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dcc620\u003e: \n failed to create HCP cluster cluster-hs-pre-drcr6j: failed waiting for cluster=\"cluster-hs-pre-drcr6j\" in resourcegroup=\"rg-hs-pre--c84xq6vfx9vp\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fb641809-c31d-45a1-b698-88b1574a5864\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fb641809-c31d-45a1-b698-88b1574a5864\",\n \"name\": \"fb641809-c31d-45a1-b698-88b1574a5864\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:32.693254211Z\",\n \"endTime\": \"2026-08-24T17:47:34.559332583Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--c84xq6vfx9vp/cluster-hs-pre-drcr6j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dcc620\u003e: \n failed to create HCP cluster cluster-hs-pre-drcr6j: failed waiting for cluster=\"cluster-hs-pre-drcr6j\" in resourcegroup=\"rg-hs-pre--c84xq6vfx9vp\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fb641809-c31d-45a1-b698-88b1574a5864\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fb641809-c31d-45a1-b698-88b1574a5864\",\n \"name\": \"fb641809-c31d-45a1-b698-88b1574a5864\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:32.693254211Z\",\n \"endTime\": \"2026-08-24T17:47:34.559332583Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9a4422a9ae8da9812507551cee69ee3f49ddb17ddb1a4021862075bf468bb85e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"e2e","signature_id":"28a8b869f35a9cd1794a8e197fc2255f15a11da6d73b85987e232efa1871d3dc","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017fe160\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-w9jj96cfm4kd\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/925e4a5c-4a74-40da-88a8-0093ce630809\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/925e4a5c-4a74-40da-88a8-0093ce630809\",\n \"name\": \"925e4a5c-4a74-40da-88a8-0093ce630809\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:33.595393096Z\",\n \"endTime\": \"2026-08-24T17:47:34.559744671Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017fe160\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-w9jj96cfm4kd\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/925e4a5c-4a74-40da-88a8-0093ce630809\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/925e4a5c-4a74-40da-88a8-0093ce630809\",\n \"name\": \"925e4a5c-4a74-40da-88a8-0093ce630809\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:33.595393096Z\",\n \"endTime\": \"2026-08-24T17:47:34.559744671Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a425edb87735a82c1566b01b5400cd18eeaa664215545b245c0b4aa096bbf181","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"e2e","signature_id":"e79e05105a6dd37d39aa319e6b82dc1c85c13b7322507d230499b9d98521e94c","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00123e240\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2xcnpkr48kxk\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3185d8c9-e246-4082-b170-898da28192d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3185d8c9-e246-4082-b170-898da28192d1\",\n \"name\": \"3185d8c9-e246-4082-b170-898da28192d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:32.888311428Z\",\n \"endTime\": \"2026-08-24T17:47:34.534131157Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00123e240\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2xcnpkr48kxk\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3185d8c9-e246-4082-b170-898da28192d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3185d8c9-e246-4082-b170-898da28192d1\",\n \"name\": \"3185d8c9-e246-4082-b170-898da28192d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:32.888311428Z\",\n \"endTime\": \"2026-08-24T17:47:34.534131157Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c4b3c1cab98cc24e4b89be210ee3e213f374451eba8553ae7925d8dc365fb100","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"e2e","signature_id":"9137a39c80626f35b65cbd5b530df9d8f5ceed8a3ed523d23d20fe02b5217e4f","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010e6ae0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-85shlfj9dj9q\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f6727d04-f4f1-41ce-9763-ca7ee1e46ac5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f6727d04-f4f1-41ce-9763-ca7ee1e46ac5\",\n \"name\": \"f6727d04-f4f1-41ce-9763-ca7ee1e46ac5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:29:34.226726362Z\",\n \"endTime\": \"2026-08-24T17:48:34.53600893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010e6ae0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-85shlfj9dj9q\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f6727d04-f4f1-41ce-9763-ca7ee1e46ac5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f6727d04-f4f1-41ce-9763-ca7ee1e46ac5\",\n \"name\": \"f6727d04-f4f1-41ce-9763-ca7ee1e46ac5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:29:34.226726362Z\",\n \"endTime\": \"2026-08-24T17:48:34.53600893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ce5fe1c26b61e2e87c186e94686621a860b7df8deb4bc8ac772833e7dd731453","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"e2e","signature_id":"9865c7135fd03ceb69a76994157b6614c9b64d8eebb69b3b00180111a55f75a4","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001b88230\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[17:51:07.885]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:51:04.0328787Z);\\nlet _endTime = datetime(2026-08-24T17:51:04.0328790Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6r7jxj55gs7j'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:51:07.8639099Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6e885cb8-5f9f-4176-bf4b-a6104ddaeca6\\\",\\r\\n \\\"activityId\\\": \\\"2b6dc4b8-71a6-45c2-8c84-a98044fb6c9c\\\",\\r\\n \\\"subActivityId\\\": \\\"dc466457-5534-436d-b9ae-dc5fd54f794d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"77ac7d84-b914-471e-9b71-1a82ec5cc8a3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e885cb8-5f9f-4176-bf4b-a6104ddaeca6 ARID=2b6dc4b8-71a6-45c2-8c84-a98044fb6c9c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f011e5f0-7470-4963-a09c-e21fd80fe1b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f011e5f0-7470-4963-a09c-e21fd80fe1b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/77ac7d84-b914-471e-9b71-1a82ec5cc8a3 \\u003e DN.FE.ExecuteQuery/dc466457-5534-436d-b9ae-dc5fd54f794d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[17:51:07.942]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:51:04.0328787Z);\\nlet _endTime = datetime(2026-08-24T17:51:04.0328790Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6r7jxj55gs7j'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:51:07.8639099Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6e885cb8-5f9f-4176-bf4b-a6104ddaeca6\\\",\\r\\n \\\"activityId\\\": \\\"2b6dc4b8-71a6-45c2-8c84-a98044fb6c9c\\\",\\r\\n \\\"subActivityId\\\": \\\"dc466457-5534-436d-b9ae-dc5fd54f794d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"77ac7d84-b914-471e-9b71-1a82ec5cc8a3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e885cb8-5f9f-4176-bf4b-a6104ddaeca6 ARID=2b6dc4b8-71a6-45c2-8c84-a98044fb6c9c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f011e5f0-7470-4963-a09c-e21fd80fe1b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f011e5f0-7470-4963-a09c-e21fd80fe1b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/77ac7d84-b914-471e-9b71-1a82ec5cc8a3 \\u003e DN.FE.ExecuteQuery/dc466457-5534-436d-b9ae-dc5fd54f794d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[17:51:11.812]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:51:08.0060747Z);\\nlet _endTime = datetime(2026-08-24T17:51:08.0060750Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6r7jxj55gs7j'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:51:11.7874017Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e5b94b99-89fd-40ca-83cf-ccc36b30573b\\\",\\r\\n \\\"activityId\\\": \\\"580e6639-eed5-43ed-b0d5-a73c66a2b85e\\\",\\r\\n \\\"subActivityId\\\": \\\"4b5d648b-cb9f-4310-a66c-e93ce0e38547\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2e63fd90-d856-490b-b345-ef00054c44aa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e5b94b99-89fd-40ca-83cf-ccc36b30573b ARID=580e6639-eed5-43ed-b0d5-a73c66a2b85e \\u003e GW.Http.CallContext/580e6639-eed5-43ed-b0d5-a73c66a2b85e \\u003e GW.ExecuteQuery/2e63fd90-d856-490b-b345-ef00054c44aa \\u003e DN.FE.ExecuteQuery/4b5d648b-cb9f-4310-a66c-e93ce0e38547)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[17:51:11.896]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:51:08.0060747Z);\\nlet _endTime = datetime(2026-08-24T17:51:08.0060750Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6r7jxj55gs7j'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:51:11.7874017Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e5b94b99-89fd-40ca-83cf-ccc36b30573b\\\",\\r\\n \\\"activityId\\\": \\\"580e6639-eed5-43ed-b0d5-a73c66a2b85e\\\",\\r\\n \\\"subActivityId\\\": \\\"4b5d648b-cb9f-4310-a66c-e93ce0e38547\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2e63fd90-d856-490b-b345-ef00054c44aa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e5b94b99-89fd-40ca-83cf-ccc36b30573b ARID=580e6639-eed5-43ed-b0d5-a73c66a2b85e \\u003e GW.Http.CallContext/580e6639-eed5-43ed-b0d5-a73c66a2b85e \\u003e GW.ExecuteQuery/2e63fd90-d856-490b-b345-ef00054c44aa \\u003e DN.FE.ExecuteQuery/4b5d648b-cb9f-4310-a66c-e93ce0e38547)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[17:51:19.503]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:51:15.9721690Z);\\nlet _endTime = datetime(2026-08-24T17:51:15.9721694Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6r7jxj55gs7j'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:51:19.4777940Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c86aa40d-117d-4a61-a0bd-780e2aa3a1d5\\\",\\r\\n \\\"activityId\\\": \\\"a72d1187-0185-4d6b-b98b-6308c9ee1cbc\\\",\\r\\n \\\"subActivityId\\\": \\\"9c08dcb8-ca10-470a-b3dc-81e2eb738b18\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b23cbcab-d834-4c81-85fe-cbbda2dc2556\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c86aa40d-117d-4a61-a0bd-780e2aa3a1d5 ARID=a72d1187-0185-4d6b-b98b-6308c9ee1cbc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9083dda2-6e55-416d-bec2-c70acc01aed8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9083dda2-6e55-416d-bec2-c70acc01aed8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/b23cbcab-d834-4c81-85fe-cbbda2dc2556 \\u003e DN.FE.ExecuteQuery/9c08dcb8-ca10-470a-b3dc-81e2eb738b18)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[17:51:19.625]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:51:15.9721690Z);\\nlet _endTime = datetime(2026-08-24T17:51:15.9721694Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6r7jxj55gs7j'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:51:19.4777940Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c86aa40d-117d-4a61-a0bd-780e2aa3a1d5\\\",\\r\\n \\\"activityId\\\": \\\"a72d1187-0185-4d6b-b98b-6308c9ee1cbc\\\",\\r\\n \\\"subActivityId\\\": \\\"9c08dcb8-ca10-470a-b3dc-81e2eb738b18\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b23cbcab-d834-4c81-85fe-cbbda2dc2556\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c86aa40d-117d-4a61-a0bd-780e2aa3a1d5 ARID=a72d1187-0185-4d6b-b98b-6308c9ee1cbc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9083dda2-6e55-416d-bec2-c70acc01aed8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9083dda2-6e55-416d-bec2-c70acc01aed8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/b23cbcab-d834-4c81-85fe-cbbda2dc2556 \\u003e DN.FE.ExecuteQuery/9c08dcb8-ca10-470a-b3dc-81e2eb738b18)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001b88230\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[17:51:07.885]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:51:04.0328787Z);\\nlet _endTime = datetime(2026-08-24T17:51:04.0328790Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6r7jxj55gs7j'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:51:07.8639099Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6e885cb8-5f9f-4176-bf4b-a6104ddaeca6\\\",\\r\\n \\\"activityId\\\": \\\"2b6dc4b8-71a6-45c2-8c84-a98044fb6c9c\\\",\\r\\n \\\"subActivityId\\\": \\\"dc466457-5534-436d-b9ae-dc5fd54f794d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"77ac7d84-b914-471e-9b71-1a82ec5cc8a3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e885cb8-5f9f-4176-bf4b-a6104ddaeca6 ARID=2b6dc4b8-71a6-45c2-8c84-a98044fb6c9c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f011e5f0-7470-4963-a09c-e21fd80fe1b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f011e5f0-7470-4963-a09c-e21fd80fe1b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/77ac7d84-b914-471e-9b71-1a82ec5cc8a3 \\u003e DN.FE.ExecuteQuery/dc466457-5534-436d-b9ae-dc5fd54f794d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[17:51:07.942]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:51:04.0328787Z);\\nlet _endTime = datetime(2026-08-24T17:51:04.0328790Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6r7jxj55gs7j'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:51:07.8639099Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;6e885cb8-5f9f-4176-bf4b-a6104ddaeca6\\\",\\r\\n \\\"activityId\\\": \\\"2b6dc4b8-71a6-45c2-8c84-a98044fb6c9c\\\",\\r\\n \\\"subActivityId\\\": \\\"dc466457-5534-436d-b9ae-dc5fd54f794d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"77ac7d84-b914-471e-9b71-1a82ec5cc8a3\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e885cb8-5f9f-4176-bf4b-a6104ddaeca6 ARID=2b6dc4b8-71a6-45c2-8c84-a98044fb6c9c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f011e5f0-7470-4963-a09c-e21fd80fe1b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/f011e5f0-7470-4963-a09c-e21fd80fe1b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/77ac7d84-b914-471e-9b71-1a82ec5cc8a3 \\u003e DN.FE.ExecuteQuery/dc466457-5534-436d-b9ae-dc5fd54f794d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[17:51:11.812]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:51:08.0060747Z);\\nlet _endTime = datetime(2026-08-24T17:51:08.0060750Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6r7jxj55gs7j'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:51:11.7874017Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e5b94b99-89fd-40ca-83cf-ccc36b30573b\\\",\\r\\n \\\"activityId\\\": \\\"580e6639-eed5-43ed-b0d5-a73c66a2b85e\\\",\\r\\n \\\"subActivityId\\\": \\\"4b5d648b-cb9f-4310-a66c-e93ce0e38547\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2e63fd90-d856-490b-b345-ef00054c44aa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e5b94b99-89fd-40ca-83cf-ccc36b30573b ARID=580e6639-eed5-43ed-b0d5-a73c66a2b85e \\u003e GW.Http.CallContext/580e6639-eed5-43ed-b0d5-a73c66a2b85e \\u003e GW.ExecuteQuery/2e63fd90-d856-490b-b345-ef00054c44aa \\u003e DN.FE.ExecuteQuery/4b5d648b-cb9f-4310-a66c-e93ce0e38547)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[17:51:11.896]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:51:08.0060747Z);\\nlet _endTime = datetime(2026-08-24T17:51:08.0060750Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6r7jxj55gs7j'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:51:11.7874017Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;e5b94b99-89fd-40ca-83cf-ccc36b30573b\\\",\\r\\n \\\"activityId\\\": \\\"580e6639-eed5-43ed-b0d5-a73c66a2b85e\\\",\\r\\n \\\"subActivityId\\\": \\\"4b5d648b-cb9f-4310-a66c-e93ce0e38547\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"2e63fd90-d856-490b-b345-ef00054c44aa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e5b94b99-89fd-40ca-83cf-ccc36b30573b ARID=580e6639-eed5-43ed-b0d5-a73c66a2b85e \\u003e GW.Http.CallContext/580e6639-eed5-43ed-b0d5-a73c66a2b85e \\u003e GW.ExecuteQuery/2e63fd90-d856-490b-b345-ef00054c44aa \\u003e DN.FE.ExecuteQuery/4b5d648b-cb9f-4310-a66c-e93ce0e38547)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[17:51:19.503]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:51:15.9721690Z);\\nlet _endTime = datetime(2026-08-24T17:51:15.9721694Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6r7jxj55gs7j'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:51:19.4777940Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c86aa40d-117d-4a61-a0bd-780e2aa3a1d5\\\",\\r\\n \\\"activityId\\\": \\\"a72d1187-0185-4d6b-b98b-6308c9ee1cbc\\\",\\r\\n \\\"subActivityId\\\": \\\"9c08dcb8-ca10-470a-b3dc-81e2eb738b18\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b23cbcab-d834-4c81-85fe-cbbda2dc2556\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c86aa40d-117d-4a61-a0bd-780e2aa3a1d5 ARID=a72d1187-0185-4d6b-b98b-6308c9ee1cbc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9083dda2-6e55-416d-bec2-c70acc01aed8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9083dda2-6e55-416d-bec2-c70acc01aed8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/b23cbcab-d834-4c81-85fe-cbbda2dc2556 \\u003e DN.FE.ExecuteQuery/9c08dcb8-ca10-470a-b3dc-81e2eb738b18)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[17:51:19.625]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:51:15.9721690Z);\\nlet _endTime = datetime(2026-08-24T17:51:15.9721694Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-6r7jxj55gs7j'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:51:19.4777940Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;c86aa40d-117d-4a61-a0bd-780e2aa3a1d5\\\",\\r\\n \\\"activityId\\\": \\\"a72d1187-0185-4d6b-b98b-6308c9ee1cbc\\\",\\r\\n \\\"subActivityId\\\": \\\"9c08dcb8-ca10-470a-b3dc-81e2eb738b18\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"b23cbcab-d834-4c81-85fe-cbbda2dc2556\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;c86aa40d-117d-4a61-a0bd-780e2aa3a1d5 ARID=a72d1187-0185-4d6b-b98b-6308c9ee1cbc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9083dda2-6e55-416d-bec2-c70acc01aed8 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/9083dda2-6e55-416d-bec2-c70acc01aed8 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/b23cbcab-d834-4c81-85fe-cbbda2dc2556 \\u003e DN.FE.ExecuteQuery/9c08dcb8-ca10-470a-b3dc-81e2eb738b18)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"dab46a9f91ba9fe79d2021eee257e65ad9d39afd86a7484c30df15125177999f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"e2e","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"e62963ca708484cd9a2b6228ea87a92725917b6e20cef794839589cf8eef9fce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","failed_at":"e2e","signature_id":"57e93beced19ad75ed8576f3f0efd935e4fd8431d2ed07db01840f4e2fefa928","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-ft86lhtlb8k7/cluster-candidate-4-20-zsqmmf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013e6360\u003e: \n failed to create HCP cluster cluster-candidate-4-20-zsqmmf: failed waiting for cluster=\"cluster-candidate-4-20-zsqmmf\" in resourcegroup=\"rg-candidate-4-20-ft86lhtlb8k7\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52586d05-b59b-44b4-8d37-e62438c994b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52586d05-b59b-44b4-8d37-e62438c994b8\",\n \"name\": \"52586d05-b59b-44b4-8d37-e62438c994b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:33.090044167Z\",\n \"endTime\": \"2026-08-24T17:47:34.534082805Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-ft86lhtlb8k7/cluster-candidate-4-20-zsqmmf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013e6360\u003e: \n failed to create HCP cluster cluster-candidate-4-20-zsqmmf: failed waiting for cluster=\"cluster-candidate-4-20-zsqmmf\" in resourcegroup=\"rg-candidate-4-20-ft86lhtlb8k7\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52586d05-b59b-44b4-8d37-e62438c994b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52586d05-b59b-44b4-8d37-e62438c994b8\",\n \"name\": \"52586d05-b59b-44b4-8d37-e62438c994b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T17:28:33.090044167Z\",\n \"endTime\": \"2026-08-24T17:47:34.534082805Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","bff39f67862ef72eccdd3c2006d48756f417856635339107ace9fde9b0a679a4","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928527292076032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4566,"pr_state":"open","pr_sha":"9749b2d351ded24fec2bb28c4ddc42fbadee5a36","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T16:32:12Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0ad1898951694a9af2ae7001dd930be414443c847c688230dccaf003468e089b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928527292076032","occurred_at":"2026-08-24T16:32:12Z","failed_at":"provision","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","test_suite":"step graph","failure_text":"time=2026-08-24T17:20:28.034Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 description=\"Step output\\n Kind: ARM\\n Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-24T17:20:28.426Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1\ntime=2026-08-24T17:20:28.817Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 deployment=3472b226dd1bf3f5c2c2fbd8f62937b0fe2f97cb4ca8a63168999f9ba0945d23 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2076032-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3472b226dd1bf3f5c2c2fbd8f62937b0fe2f97cb4ca8a63168999f9ba0945d23\ntime=2026-08-24T17:20:28.866Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2076032-mgmt-1/providers/Microsoft.Resources/deployments/3472b226dd1bf3f5c2c2fbd8f62937b0fe2f97cb4ca8a63168999f9ba0945d23/operationStatuses/08584140148567905734\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment '3472b226dd1bf3f5c2c2fbd8f62937b0fe2f97cb4ca8a63168999f9ba0945d23' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091926769790619648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6639,"pr_state":"open","pr_sha":"ccc0138e9d9cba8c23cc66260ecb2042998ab2a6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T16:21:04Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"aee6749443d9f050b6537618ebc84cb163f6c3a072d60a747c073428f6396f7d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091926769790619648","occurred_at":"2026-08-24T16:21:04Z","failed_at":"e2e","signature_id":"40f2371fc4b699bfbeed8a52087620105728edb5da70261d5e9bd7df99b13e6a","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0009863f0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[17:50:10.830]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:50:07.0974750Z);\\nlet _endTime = datetime(2026-08-24T17:50:07.0974754Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bsdxktxx588n'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:50:10.8083087Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;fac29151-0f82-45b0-b39e-1b80a12094a4\\\",\\r\\n \\\"activityId\\\": \\\"578602da-c7bd-4612-b601-5ddeff2f3885\\\",\\r\\n \\\"subActivityId\\\": \\\"4dee2970-8e9b-48d4-bf21-5bfe620f649d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d1fdc108-83eb-4130-b339-83dd3ff9fcbb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;fac29151-0f82-45b0-b39e-1b80a12094a4 ARID=578602da-c7bd-4612-b601-5ddeff2f3885 \\u003e GW.Http.CallContext/578602da-c7bd-4612-b601-5ddeff2f3885 \\u003e GW.ExecuteQuery/d1fdc108-83eb-4130-b339-83dd3ff9fcbb \\u003e DN.FE.ExecuteQuery/4dee2970-8e9b-48d4-bf21-5bfe620f649d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[17:50:10.904]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:50:07.0974750Z);\\nlet _endTime = datetime(2026-08-24T17:50:07.0974754Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bsdxktxx588n'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:50:10.8083087Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;fac29151-0f82-45b0-b39e-1b80a12094a4\\\",\\r\\n \\\"activityId\\\": \\\"578602da-c7bd-4612-b601-5ddeff2f3885\\\",\\r\\n \\\"subActivityId\\\": \\\"4dee2970-8e9b-48d4-bf21-5bfe620f649d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d1fdc108-83eb-4130-b339-83dd3ff9fcbb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;fac29151-0f82-45b0-b39e-1b80a12094a4 ARID=578602da-c7bd-4612-b601-5ddeff2f3885 \\u003e GW.Http.CallContext/578602da-c7bd-4612-b601-5ddeff2f3885 \\u003e GW.ExecuteQuery/d1fdc108-83eb-4130-b339-83dd3ff9fcbb \\u003e DN.FE.ExecuteQuery/4dee2970-8e9b-48d4-bf21-5bfe620f649d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[17:50:14.600]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:50:10.9799365Z);\\nlet _endTime = datetime(2026-08-24T17:50:10.9799369Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bsdxktxx588n'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:50:14.5738009Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;9cfffa2e-fd47-4de8-b850-fcb128ae3957\\\",\\r\\n \\\"activityId\\\": \\\"b1a9e4f8-fe9f-43eb-98c4-295226684f99\\\",\\r\\n \\\"subActivityId\\\": \\\"5346eafa-957d-49ee-b78d-d8e0622b2808\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ea635fed-0287-4459-9508-ca4085e8c3aa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;9cfffa2e-fd47-4de8-b850-fcb128ae3957 ARID=b1a9e4f8-fe9f-43eb-98c4-295226684f99 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/33092b63-29bb-42fc-b88a-af6bae1ef15f \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/33092b63-29bb-42fc-b88a-af6bae1ef15f \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ea635fed-0287-4459-9508-ca4085e8c3aa \\u003e DN.FE.ExecuteQuery/5346eafa-957d-49ee-b78d-d8e0622b2808)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[17:50:14.688]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:50:10.9799365Z);\\nlet _endTime = datetime(2026-08-24T17:50:10.9799369Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bsdxktxx588n'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:50:14.5738009Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;9cfffa2e-fd47-4de8-b850-fcb128ae3957\\\",\\r\\n \\\"activityId\\\": \\\"b1a9e4f8-fe9f-43eb-98c4-295226684f99\\\",\\r\\n \\\"subActivityId\\\": \\\"5346eafa-957d-49ee-b78d-d8e0622b2808\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ea635fed-0287-4459-9508-ca4085e8c3aa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;9cfffa2e-fd47-4de8-b850-fcb128ae3957 ARID=b1a9e4f8-fe9f-43eb-98c4-295226684f99 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/33092b63-29bb-42fc-b88a-af6bae1ef15f \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/33092b63-29bb-42fc-b88a-af6bae1ef15f \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ea635fed-0287-4459-9508-ca4085e8c3aa \\u003e DN.FE.ExecuteQuery/5346eafa-957d-49ee-b78d-d8e0622b2808)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[17:50:21.769]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:50:18.2612887Z);\\nlet _endTime = datetime(2026-08-24T17:50:18.2612891Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bsdxktxx588n'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:50:21.7433795Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8abde100-7c98-4f34-a0a5-8f66c337da8e\\\",\\r\\n \\\"activityId\\\": \\\"a71d2be1-e393-45cd-a6e4-5527dafdf569\\\",\\r\\n \\\"subActivityId\\\": \\\"71101ab2-ff83-4119-8e2b-ee21aaceb2d6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"dcbe4a01-596d-4348-affe-223b58329c91\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8abde100-7c98-4f34-a0a5-8f66c337da8e ARID=a71d2be1-e393-45cd-a6e4-5527dafdf569 \\u003e GW.Http.CallContext/a71d2be1-e393-45cd-a6e4-5527dafdf569 \\u003e GW.ExecuteQuery/dcbe4a01-596d-4348-affe-223b58329c91 \\u003e DN.FE.ExecuteQuery/71101ab2-ff83-4119-8e2b-ee21aaceb2d6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[17:50:21.932]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:50:18.2612887Z);\\nlet _endTime = datetime(2026-08-24T17:50:18.2612891Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bsdxktxx588n'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:50:21.7433795Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8abde100-7c98-4f34-a0a5-8f66c337da8e\\\",\\r\\n \\\"activityId\\\": \\\"a71d2be1-e393-45cd-a6e4-5527dafdf569\\\",\\r\\n \\\"subActivityId\\\": \\\"71101ab2-ff83-4119-8e2b-ee21aaceb2d6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"dcbe4a01-596d-4348-affe-223b58329c91\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8abde100-7c98-4f34-a0a5-8f66c337da8e ARID=a71d2be1-e393-45cd-a6e4-5527dafdf569 \\u003e GW.Http.CallContext/a71d2be1-e393-45cd-a6e4-5527dafdf569 \\u003e GW.ExecuteQuery/dcbe4a01-596d-4348-affe-223b58329c91 \\u003e DN.FE.ExecuteQuery/71101ab2-ff83-4119-8e2b-ee21aaceb2d6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0009863f0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[17:50:10.830]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:50:07.0974750Z);\\nlet _endTime = datetime(2026-08-24T17:50:07.0974754Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bsdxktxx588n'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:50:10.8083087Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;fac29151-0f82-45b0-b39e-1b80a12094a4\\\",\\r\\n \\\"activityId\\\": \\\"578602da-c7bd-4612-b601-5ddeff2f3885\\\",\\r\\n \\\"subActivityId\\\": \\\"4dee2970-8e9b-48d4-bf21-5bfe620f649d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d1fdc108-83eb-4130-b339-83dd3ff9fcbb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;fac29151-0f82-45b0-b39e-1b80a12094a4 ARID=578602da-c7bd-4612-b601-5ddeff2f3885 \\u003e GW.Http.CallContext/578602da-c7bd-4612-b601-5ddeff2f3885 \\u003e GW.ExecuteQuery/d1fdc108-83eb-4130-b339-83dd3ff9fcbb \\u003e DN.FE.ExecuteQuery/4dee2970-8e9b-48d4-bf21-5bfe620f649d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[17:50:10.904]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:50:07.0974750Z);\\nlet _endTime = datetime(2026-08-24T17:50:07.0974754Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bsdxktxx588n'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:50:10.8083087Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;fac29151-0f82-45b0-b39e-1b80a12094a4\\\",\\r\\n \\\"activityId\\\": \\\"578602da-c7bd-4612-b601-5ddeff2f3885\\\",\\r\\n \\\"subActivityId\\\": \\\"4dee2970-8e9b-48d4-bf21-5bfe620f649d\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"d1fdc108-83eb-4130-b339-83dd3ff9fcbb\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;fac29151-0f82-45b0-b39e-1b80a12094a4 ARID=578602da-c7bd-4612-b601-5ddeff2f3885 \\u003e GW.Http.CallContext/578602da-c7bd-4612-b601-5ddeff2f3885 \\u003e GW.ExecuteQuery/d1fdc108-83eb-4130-b339-83dd3ff9fcbb \\u003e DN.FE.ExecuteQuery/4dee2970-8e9b-48d4-bf21-5bfe620f649d)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[17:50:14.600]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:50:10.9799365Z);\\nlet _endTime = datetime(2026-08-24T17:50:10.9799369Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bsdxktxx588n'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:50:14.5738009Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;9cfffa2e-fd47-4de8-b850-fcb128ae3957\\\",\\r\\n \\\"activityId\\\": \\\"b1a9e4f8-fe9f-43eb-98c4-295226684f99\\\",\\r\\n \\\"subActivityId\\\": \\\"5346eafa-957d-49ee-b78d-d8e0622b2808\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ea635fed-0287-4459-9508-ca4085e8c3aa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;9cfffa2e-fd47-4de8-b850-fcb128ae3957 ARID=b1a9e4f8-fe9f-43eb-98c4-295226684f99 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/33092b63-29bb-42fc-b88a-af6bae1ef15f \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/33092b63-29bb-42fc-b88a-af6bae1ef15f \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ea635fed-0287-4459-9508-ca4085e8c3aa \\u003e DN.FE.ExecuteQuery/5346eafa-957d-49ee-b78d-d8e0622b2808)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[17:50:14.688]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:50:10.9799365Z);\\nlet _endTime = datetime(2026-08-24T17:50:10.9799369Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bsdxktxx588n'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:50:14.5738009Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;9cfffa2e-fd47-4de8-b850-fcb128ae3957\\\",\\r\\n \\\"activityId\\\": \\\"b1a9e4f8-fe9f-43eb-98c4-295226684f99\\\",\\r\\n \\\"subActivityId\\\": \\\"5346eafa-957d-49ee-b78d-d8e0622b2808\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"ea635fed-0287-4459-9508-ca4085e8c3aa\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;9cfffa2e-fd47-4de8-b850-fcb128ae3957 ARID=b1a9e4f8-fe9f-43eb-98c4-295226684f99 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/33092b63-29bb-42fc-b88a-af6bae1ef15f \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/33092b63-29bb-42fc-b88a-af6bae1ef15f \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/ea635fed-0287-4459-9508-ca4085e8c3aa \\u003e DN.FE.ExecuteQuery/5346eafa-957d-49ee-b78d-d8e0622b2808)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[17:50:21.769]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:50:18.2612887Z);\\nlet _endTime = datetime(2026-08-24T17:50:18.2612891Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bsdxktxx588n'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:50:21.7433795Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8abde100-7c98-4f34-a0a5-8f66c337da8e\\\",\\r\\n \\\"activityId\\\": \\\"a71d2be1-e393-45cd-a6e4-5527dafdf569\\\",\\r\\n \\\"subActivityId\\\": \\\"71101ab2-ff83-4119-8e2b-ee21aaceb2d6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"dcbe4a01-596d-4348-affe-223b58329c91\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8abde100-7c98-4f34-a0a5-8f66c337da8e ARID=a71d2be1-e393-45cd-a6e4-5527dafdf569 \\u003e GW.Http.CallContext/a71d2be1-e393-45cd-a6e4-5527dafdf569 \\u003e GW.ExecuteQuery/dcbe4a01-596d-4348-affe-223b58329c91 \\u003e DN.FE.ExecuteQuery/71101ab2-ff83-4119-8e2b-ee21aaceb2d6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[17:50:21.932]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-23T17:50:18.2612887Z);\\nlet _endTime = datetime(2026-08-24T17:50:18.2612891Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-bsdxktxx588n'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n \\\"@failureCode\\\": 400,\\r\\n \\\"@context\\\": {\\r\\n \\\"timestamp\\\": \\\"2026-08-24T17:50:21.7433795Z\\\",\\r\\n \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n \\\"clientRequestId\\\": \\\"KGC.execute;8abde100-7c98-4f34-a0a5-8f66c337da8e\\\",\\r\\n \\\"activityId\\\": \\\"a71d2be1-e393-45cd-a6e4-5527dafdf569\\\",\\r\\n \\\"subActivityId\\\": \\\"71101ab2-ff83-4119-8e2b-ee21aaceb2d6\\\",\\r\\n \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n \\\"parentActivityId\\\": \\\"dcbe4a01-596d-4348-affe-223b58329c91\\\",\\r\\n \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;8abde100-7c98-4f34-a0a5-8f66c337da8e ARID=a71d2be1-e393-45cd-a6e4-5527dafdf569 \\u003e GW.Http.CallContext/a71d2be1-e393-45cd-a6e4-5527dafdf569 \\u003e GW.ExecuteQuery/dcbe4a01-596d-4348-affe-223b58329c91 \\u003e DN.FE.ExecuteQuery/71101ab2-ff83-4119-8e2b-ee21aaceb2d6)\\\",\\r\\n \\\"serviceFarm\\\": null\\r\\n },\\r\\n \\\"@permanent\\\": true\\r\\n }\\r\\n}\"\n }�[0m\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091921367619342336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6638,"pr_state":"open","pr_sha":"c403afe8d857fa8c76e2da31fc1c9d3952ca11b9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T16:10:41Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"3ecc284f9b75b12385a1abd0d0dc6d60e98517aed09620576426964316608e50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091921367619342336","occurred_at":"2026-08-24T16:10:41Z","failed_at":"alert","signature_id":"d4e52f6e7208e4a023c43c3ea9a95264341c256a3dcba5d703516c2b2c81f97b","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T17:21:19Z\n Ended: 2026-08-24T17:40:17Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9342336-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9342336-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091920407983558656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6584,"pr_state":"open","pr_sha":"c9b84520822a4a33afe34017374746e70acb738e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T16:06:36Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"16bc4917194fa2c5829c30db63e09fc1122309015abd9ddcbcdf4c2a205fb586","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091920407983558656","occurred_at":"2026-08-24T16:06:36Z","failed_at":"alert","signature_id":"c52899aff7bf73e0f12d284cdf7344825014b3fd387fb2f270a8871271bc7acb","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T17:05:25Z\n Ended: 2026-08-24T17:12:23Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3558656-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd2i4t2jotejuk2memhr38s3p4vth8f-n6e5i1i6m0d7n8a\", pod=\"router-6597db7-qp7zx\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd2i4t2jotejuk2memhr38s3p4vth8f-n6e5i1i6m0d7n8a/router-6597db7-qp7zx has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"de02447d785a84a3657a3e7e36d54ddf742544621bfd054475479590c3a78b21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091920407983558656","occurred_at":"2026-08-24T16:06:36Z","failed_at":"e2e","signature_id":"fdde03128e739a2f562ab5c3485dae63a0f561b3b4659197733a62ff1ea2a4e8","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef90e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2zg88lxq9q4w\" to finish creating: GET https://rp.j3558656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\",\n \"name\": \"b976f9f3-2891-4433-868a-76a8d23de317\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T16:54:08.369328068Z\",\n \"endTime\": \"2026-08-24T17:13:08.208412685Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef90e0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2zg88lxq9q4w\" to finish creating: GET https://rp.j3558656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\",\n \"name\": \"b976f9f3-2891-4433-868a-76a8d23de317\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T16:54:08.369328068Z\",\n \"endTime\": \"2026-08-24T17:13:08.208412685Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b652815635f16d983d1325dbd7e304560647aed7cf1be6254e8a2a57faf1d4fa","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b652815635f16d983d1325dbd7e304560647aed7cf1be6254e8a2a57faf1d4fa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091917671514771456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6645,"pr_state":"open","pr_sha":"3c81c748e92c32ebefa3e5019d3bd3f86cf04336","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-24T15:56:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091917650543251456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-24T15:56:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091916494920224768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6278,"pr_state":"closed","pr_sha":"33f9055c00d2b52023c4da69e51807e8d141a7e5","final_merged_sha":"33f9055c00d2b52023c4da69e51807e8d141a7e5","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-24T15:52:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091915412907233280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6504,"pr_state":"open","pr_sha":"af0ab1f037eef8141b97aadd390cc54296b6d1c3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-24T15:47:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091914307531968512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6659,"pr_state":"open","pr_sha":"bdc899665a70aef2c2511f671093b1e22311e382","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T15:43:31Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"810097958ef24190173f7ed0fd7e0c512648e1e47845c866f05024d4b467ab85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091914307531968512","occurred_at":"2026-08-24T15:43:31Z","failed_at":"provision","signature_id":"d517e51fc5b09c819903bacd247416bcbbe167939922787f3a2438a125b3fdc2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","test_suite":"step graph","failure_text":"time=2026-08-24T16:09:49.176Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 description=\"Step infra\\n Kind: ARM\\n Template: templates/mgmt-infra.bicep\\n Parameters: configurations/mgmt-infra.tmpl.bicepparam\"\ntime=2026-08-24T16:09:49.840Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2\ntime=2026-08-24T16:09:50.553Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 deployment=c783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1968512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290\ntime=2026-08-24T16:11:20.899Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/c783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290/operationStatuses/08584140190952500816\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcp-backups-storage\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcpBackupsStorageAccount\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcpBackupsStorageAccount\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"StorageAccountOperationInProgress\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"An operation is currently performing on this storage account that requires exclusive access.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bb424b290c173083428a34ac21b9472b0a9be4d1c592b771df2c71e46b269465","failure_pattern":"ERROR CODE: DeploymentFailed; detail code StorageAccountOperationInProgress; detail message An operation is currently performing on this storage account that requires exclusive access.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bb424b290c173083428a34ac21b9472b0a9be4d1c592b771df2c71e46b269465"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091911887242399744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6363,"pr_state":"open","pr_sha":"00c069f9b0ee7d18fdba45d895bbd5c595004327","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-24T15:33:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091905673813037056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6070,"pr_state":"open","pr_sha":"2f68bd3b981a9e093b933e26b060f8e1150c0eba","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T15:09:12Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"55c2fb2e7c913aa1231f5daee75c7001e883d48029a8fc417250799920297752","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091905673813037056","occurred_at":"2026-08-24T15:09:12Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5958,"pr_state":"open","pr_sha":"8ce3ed6e8d91907caf45e2b3ec8c94ff7141fe0a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T14:50:52Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"2dabf0ce5071fef4b3e3968476a4858de288714d8f2056626bd183b029ab599c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","failed_at":"alert","signature_id":"c239e70f5bf7ad0e42856cb2b9acc6d7c10a9ba7d6c9a4f5496cbd3609446018","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-08-24T16:04:45Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n Description: Container fluentbit in pod arobit/arobit-forwarder-cxwt8 on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-08-24T16:04:46Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-jbtcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1ec91183-7c07-473b-b1d1-9df325a61c81\"\n Description: Container fluentbit in pod arobit/arobit-forwarder-jbtcb on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-08-24T16:04:46Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-jbtcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1ec91183-7c07-473b-b1d1-9df325a61c81\"\n Description: Container fluentbit in pod arobit/arobit-forwarder-jbtcb on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-08-24T16:04:46Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n Description: Container fluentbit in pod arobit/arobit-forwarder-cxwt8 on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired"}},{"row_id":"667dff269b39b8e1892469ef6cbe164532e23a78c766a6ce918c1c4c4795671c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","failed_at":"alert","signature_id":"367941eb104ee306ff1fe196c961bc8c4a584d69e0f31a0ee0696e981fbdf639","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 24 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T16:11:08Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:25:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-08-24T16:13:07Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-08-24T16:13:07Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:26:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:24:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-08-24T16:17:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-08-24T16:17:09Z\n Ended: 2026-08-24T16:23:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-08-24T16:17:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-08-24T16:17:09Z\n Ended: 2026-08-24T16:29:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"696d3165dbca3269e7ca6eb85a88b2fb50203273d82684effadfc57ca80823ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","failed_at":"e2e","signature_id":"7dd423d6ebe7dc246942b7ca078647aad56aaec8baea7f2d4aa8292a909f5f19","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e70e20\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wmkndtkx988t\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\",\n \"name\": \"a9d4baef-549e-4561-b748-439453d113af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:49:08.39871397Z\",\n \"endTime\": \"2026-08-24T16:08:11.236941815Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e70e20\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wmkndtkx988t\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\",\n \"name\": \"a9d4baef-549e-4561-b748-439453d113af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:49:08.39871397Z\",\n \"endTime\": \"2026-08-24T16:08:11.236941815Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4a2f8bcf8d29c66434783789d75f6d18e6e94158f0e0f35e53c0353bce508345","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"71f83a7376bd5fdf3ff584311d5d2a309124d245cf3bc46613d38d62f01114fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","failed_at":"e2e","signature_id":"d07feba14b1358e1ba91e00ab3fcc20b89813cd03b2520364e81ecc050b70a83","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000408ec0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn9cdfvxcqll\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\",\n \"name\": \"75726762-be88-402f-a564-60c91125dc8b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:47:28.10624954Z\",\n \"endTime\": \"2026-08-24T16:06:31.149377863Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000408ec0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn9cdfvxcqll\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\",\n \"name\": \"75726762-be88-402f-a564-60c91125dc8b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:47:28.10624954Z\",\n \"endTime\": \"2026-08-24T16:06:31.149377863Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4a2f8bcf8d29c66434783789d75f6d18e6e94158f0e0f35e53c0353bce508345","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"76e2cbc167e20a0c8ff049e092852e8ba871e0b0eadb8819bc1b0bd0471513b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","failed_at":"alert","signature_id":"fe91f4f69f02671271e4cc6e4aaa4f966388d16d9e660f0bf8e6f1f0d5179d16","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T16:19:07Z\n Ended: 2026-08-24T16:30:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n Description: Pod arobit/arobit-forwarder-cxwt8 (fluentbit) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-08-24T16:20:06Z\n Ended: 2026-08-24T16:33:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n Description: Pod arobit/arobit-forwarder-cxwt8 (fluentbit) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"8d85045dd619baabf971fd9958504ec6683702b88017f2f1977e98f596ec6a77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","failed_at":"alert","signature_id":"8ea4612bb5240c0f449b27c94dc170f61dc652ef24bc74880e6489a1d15ea2be","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T16:12:59Z\n Ended: 2026-08-24T16:22:56Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"prometheus\", node=\"aks-userswft2-20578738-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: aks-userswft2-20578738-vmss000002 is unreachable and some workloads may be rescheduled.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T16:12:59Z\n Ended: 2026-08-24T16:23:55Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"prometheus\", node=\"aks-userswft2-20578738-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: aks-userswft2-20578738-vmss000002 is unreachable and some workloads may be rescheduled.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","failure_pattern":"alert [svc] KubeNodeUnreachable fired","search_query":"alert [svc] KubeNodeUnreachable fired"}},{"row_id":"8e0df42401573b3dac69b0e0ed2c8f0f1d59842b5a2d4cfbd2cdf5597407bac7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","failed_at":"alert","signature_id":"7caa0010541a7c4ff937ecc7f1700635aa06e8c5ef418d2f4d2c09799f427a78","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T16:12:09Z\n Ended: 2026-08-24T16:24:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 3:\n State: Resolved\n Started: 2026-08-24T16:13:10Z\n Ended: 2026-08-24T16:23:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 4:\n State: Resolved\n Started: 2026-08-24T16:13:11Z\n Ended: 2026-08-24T16:23:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","failure_pattern":"alert [svc] KubeDaemonSetMisScheduled fired","search_query":"alert [svc] KubeDaemonSetMisScheduled fired"}},{"row_id":"ef24e0682adcbc6f689d4e1e4643543025391361dcd4715bf22046d3bb65932d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","failed_at":"e2e","signature_id":"371e2dd52ef73c97161118e4d4efe23b9ace8422b8af743704455c3d7faec23b","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["4a2f8bcf8d29c66434783789d75f6d18e6e94158f0e0f35e53c0353bce508345","519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6629,"pr_state":"open","pr_sha":"c6d46781a7abcc4c551cdf9edbb601d3aee0cc38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T14:44:27Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"010bf85a2c9dde58fa646793ab9b1908ef4e13740de9b429a93f89ddee240bf4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","failed_at":"e2e","signature_id":"fd285a3626d258dcb1ba2d99b9eb4762dc5e5af32183bf400bd799c872dd2fda","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015a4b20\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-pdzrhq4kn5dh\" to finish creating: GET https://rp.j9446912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3839a4cf-1d74-435a-ad7d-cf4ad8e2f726\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3839a4cf-1d74-435a-ad7d-cf4ad8e2f726\",\n \"name\": \"3839a4cf-1d74-435a-ad7d-cf4ad8e2f726\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:35:46.321920583Z\",\n \"endTime\": \"2026-08-24T15:54:46.656362714Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015a4b20\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-pdzrhq4kn5dh\" to finish creating: GET https://rp.j9446912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3839a4cf-1d74-435a-ad7d-cf4ad8e2f726\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3839a4cf-1d74-435a-ad7d-cf4ad8e2f726\",\n \"name\": \"3839a4cf-1d74-435a-ad7d-cf4ad8e2f726\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:35:46.321920583Z\",\n \"endTime\": \"2026-08-24T15:54:46.656362714Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2d76d240c1720853439f4a72d6b6d99c9388a9c6b228dd1d6f05d195ea5ce8bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","failed_at":"e2e","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"45ba062db451305625c740146382da171b0d95995154b76f0386e461e3ef9fdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","failed_at":"alert","signature_id":"32b177871d4669aa26bebdb303a13d4eeafa32be58fe0c29eecd2b7355b315ff","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T15:48:52Z\n Ended: 2026-08-24T16:05:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9446912-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd1dkicrfebg5gk7mjo71hbgeofv807-s9h5a0t6o1q6t2p\", pod=\"router-6dd6c76467-mz8vb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd1dkicrfebg5gk7mjo71hbgeofv807-s9h5a0t6o1q6t2p/router-6dd6c76467-mz8vb has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"bbcd62b0b520ecdd020cdf24fc592a829392c0561bdf69625196536088fefb9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","failed_at":"e2e","signature_id":"0119ef26d501844a6f69a87b74e13b68dec77db9c3af742f0d6e0ca5c74e1853","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001ab4ac0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6b45927pz45x\" to finish creating: GET https://rp.j9446912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f98ab769-8943-4e16-9ba2-6234424afc6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f98ab769-8943-4e16-9ba2-6234424afc6d\",\n \"name\": \"f98ab769-8943-4e16-9ba2-6234424afc6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:35:32.53710884Z\",\n \"endTime\": \"2026-08-24T15:54:36.487936419Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001ab4ac0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6b45927pz45x\" to finish creating: GET https://rp.j9446912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f98ab769-8943-4e16-9ba2-6234424afc6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f98ab769-8943-4e16-9ba2-6234424afc6d\",\n \"name\": \"f98ab769-8943-4e16-9ba2-6234424afc6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:35:32.53710884Z\",\n \"endTime\": \"2026-08-24T15:54:36.487936419Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"3ec9ecd4e53729b55fe8f972a0acc75827b54561","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T14:42:21Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"2e5baa6c83273d8279c4f8de47daf275506413aefe4d6bcb86df7b8892b76cff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","failed_at":"e2e","signature_id":"94f9ba782dfa48d054a8eb8de04c3bde301c62297127cae4d03aeb5f65b1b3b4","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:89]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00182e2a0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-k4d2bsmbhrnc\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3eb4e8a-5daa-40ba-be6e-b23c10b4eafe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3eb4e8a-5daa-40ba-be6e-b23c10b4eafe\",\n \"name\": \"d3eb4e8a-5daa-40ba-be6e-b23c10b4eafe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:41:38.68937252Z\",\n \"endTime\": \"2026-08-24T16:00:39.082023736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:89]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00182e2a0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-k4d2bsmbhrnc\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3eb4e8a-5daa-40ba-be6e-b23c10b4eafe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3eb4e8a-5daa-40ba-be6e-b23c10b4eafe\",\n \"name\": \"d3eb4e8a-5daa-40ba-be6e-b23c10b4eafe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:41:38.68937252Z\",\n \"endTime\": \"2026-08-24T16:00:39.082023736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"303a609e15bf39945e30740ea4beed9d34cac2a50deb0168328b750a695974c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","failed_at":"alert","signature_id":"d1d5a7d0ff60a9161bf6b8e8b3cd897227641812139ecab6ded5ff874a074bf7","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s) (2 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T15:55:28Z\n Ended: 2026-08-24T16:09:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8539776-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd1ghoa165uh75qt39elkm018udft1l-u7o1r1e3a1i8p2h\", pod=\"router-bc5b66df4-m2tkb\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd1ghoa165uh75qt39elkm018udft1l-u7o1r1e3a1i8p2h/router-bc5b66df4-m2tkb has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T15:56:26Z\n Ended: 2026-08-24T16:09:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8539776-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd1gcvqlv2t9s69eq6f5mdfd7qjbb5f-h8k0a3a7f2f5p1y\", pod=\"router-7b6dfdd9b-2c6nj\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd1gcvqlv2t9s69eq6f5mdfd7qjbb5f-h8k0a3a7f2f5p1y/router-7b6dfdd9b-2c6nj has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"943a8ba186022e6ad0fe625f86b257322fffa29aebf12d8414e91ee544e2dd4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","failed_at":"e2e","signature_id":"7f0730f0d1c7e0eb7a035ce2e1a824a7fb99f16bf281917ec8e0ea592b939bdd","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:726]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd0320\u003e: \n failed to create HCP cluster np-dg-minor-4tds4q: failed waiting for cluster=\"np-dg-minor-4tds4q\" in resourcegroup=\"rg-np-dg-minor-4tds4q-6qtw9l2fwcdn\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/162a10b5-886b-40a8-a95b-30fc38ce9bc5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/162a10b5-886b-40a8-a95b-30fc38ce9bc5\",\n \"name\": \"162a10b5-886b-40a8-a95b-30fc38ce9bc5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:41:41.593852701Z\",\n \"endTime\": \"2026-08-24T16:00:49.106385261Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:726]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd0320\u003e: \n failed to create HCP cluster np-dg-minor-4tds4q: failed waiting for cluster=\"np-dg-minor-4tds4q\" in resourcegroup=\"rg-np-dg-minor-4tds4q-6qtw9l2fwcdn\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/162a10b5-886b-40a8-a95b-30fc38ce9bc5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/162a10b5-886b-40a8-a95b-30fc38ce9bc5\",\n \"name\": \"162a10b5-886b-40a8-a95b-30fc38ce9bc5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:41:41.593852701Z\",\n \"endTime\": \"2026-08-24T16:00:49.106385261Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d1500c574faef5477ba455a9a07bb860fa2fa6142ecbc20222bb5cdee5e70b54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","failed_at":"e2e","signature_id":"e6714abbbcfe1d54c45ec38583b6367bc1667d055485de6ce77544f68acc9679","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:310]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00127e240\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-4s95f76fslkw\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a5bec99d-ae1e-4543-a8ff-8aa060972e34\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a5bec99d-ae1e-4543-a8ff-8aa060972e34\",\n \"name\": \"a5bec99d-ae1e-4543-a8ff-8aa060972e34\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:41:58.968228589Z\",\n \"endTime\": \"2026-08-24T16:00:59.046704933Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:310]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00127e240\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-4s95f76fslkw\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a5bec99d-ae1e-4543-a8ff-8aa060972e34\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a5bec99d-ae1e-4543-a8ff-8aa060972e34\",\n \"name\": \"a5bec99d-ae1e-4543-a8ff-8aa060972e34\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:41:58.968228589Z\",\n \"endTime\": \"2026-08-24T16:00:59.046704933Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e8e442eaa8f90cb066c491f21cbafd6ab9f224b370a429c86ed143d72f967a53","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","failed_at":"e2e","signature_id":"3605097c99e7a1623e2e3dd91c5b9641833e016d1c0c516d9b0275085f65863f","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:109]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa81a0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jhvt8ptm7fw8\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12485de2-db77-439f-89df-5bacb5d218b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12485de2-db77-439f-89df-5bacb5d218b2\",\n \"name\": \"12485de2-db77-439f-89df-5bacb5d218b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:41:38.67903036Z\",\n \"endTime\": \"2026-08-24T16:00:39.109568382Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:109]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa81a0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jhvt8ptm7fw8\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12485de2-db77-439f-89df-5bacb5d218b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12485de2-db77-439f-89df-5bacb5d218b2\",\n \"name\": \"12485de2-db77-439f-89df-5bacb5d218b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:41:38.67903036Z\",\n \"endTime\": \"2026-08-24T16:00:39.109568382Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091897495339667456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6643,"pr_state":"open","pr_sha":"ec049a815d7f0c06b8cbfe37e4a92aff95ec1095","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-24T14:36:42Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"54bb2db04ddf012dceee9c7b7444143151cf3825","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T14:34:02Z"},"failed_at":["alert","e2e"],"failed_test_count":7,"occurrences":[{"row_id":"0d57ec008a3f478913a7cf5b9d6260b39175dcbe8e95627735a0e7e571b73aba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","failed_at":"e2e","signature_id":"4de409ed2fffcc46ddb557100c947bf3b037028f8c648ac70391763c58835df0","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e86920\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-7vpnvrbqnldf\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73cc6203-30af-4c53-9024-398221683c83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73cc6203-30af-4c53-9024-398221683c83\",\n \"name\": \"73cc6203-30af-4c53-9024-398221683c83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:17:58.555796542Z\",\n \"endTime\": \"2026-08-24T15:37:08.01693151Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e86920\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-7vpnvrbqnldf\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73cc6203-30af-4c53-9024-398221683c83\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73cc6203-30af-4c53-9024-398221683c83\",\n \"name\": \"73cc6203-30af-4c53-9024-398221683c83\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:17:58.555796542Z\",\n \"endTime\": \"2026-08-24T15:37:08.01693151Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"116c73aba6b2d1ea9c87ce290b22b42bebacfff82a632fe6e0251ac1ab03df6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","failed_at":"e2e","signature_id":"6b56df09ac5297e89b7853c7486e42c460930ae4b037271fddfef92b6ce0846a","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a808e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-cgt6b2gspgmg\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c64a33f9-1d34-4ca6-89af-ad50a67745b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c64a33f9-1d34-4ca6-89af-ad50a67745b6\",\n \"name\": \"c64a33f9-1d34-4ca6-89af-ad50a67745b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:19:10.873312133Z\",\n \"endTime\": \"2026-08-24T15:38:17.935941947Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-network-operator, redhat-operators-catalog, catalog-operator, openshift-route-controller-manager, ingress-operator, hosted-cluster-config-operator, packageserver, cluster-policy-controller, cluster-storage-operator, certified-operators-catalog, olm-operator, openshift-controller-manager, csi-snapshot-controller-operator, community-operators-catalog, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a808e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-cgt6b2gspgmg\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c64a33f9-1d34-4ca6-89af-ad50a67745b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c64a33f9-1d34-4ca6-89af-ad50a67745b6\",\n \"name\": \"c64a33f9-1d34-4ca6-89af-ad50a67745b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:19:10.873312133Z\",\n \"endTime\": \"2026-08-24T15:38:17.935941947Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-network-operator, redhat-operators-catalog, catalog-operator, openshift-route-controller-manager, ingress-operator, hosted-cluster-config-operator, packageserver, cluster-policy-controller, cluster-storage-operator, certified-operators-catalog, olm-operator, openshift-controller-manager, csi-snapshot-controller-operator, community-operators-catalog, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7656ecefb1d0ad34a814120d4b283a9cf7e92b4d30f689a1835643bfd45aac64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","failed_at":"alert","signature_id":"41f9fa028fb65313589b4f31abf40b7c37b7c57dd66b9ac548b1148f02514739","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 14 time(s) (4 unknown, 10 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T15:32:32Z\n Ended: 2026-08-24T15:39:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1362816-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd15mfbnh569osr6v3jpmrh2u4ovste-n1x0n6k7s4n4s4m\", pod=\"etcd-0\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd15mfbnh569osr6v3jpmrh2u4ovste-n1x0n6k7s4n4s4m/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T15:33:32Z\n Ended: 2026-08-24T15:39:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1362816-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd15s2tbl2mk3taqpkr4l2ue96f7dbj-agg-cluster\", pod=\"cluster-api-7b8d6c548b-n9txh\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd15s2tbl2mk3taqpkr4l2ue96f7dbj-agg-cluster/cluster-api-7b8d6c548b-n9txh has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-24T15:33:32Z\n Ended: 2026-08-24T15:39:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1362816-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd16iscun46vl67eantdmfbrchq5cvk-q4u5f1h8q8y4t5a\", pod=\"kube-state-metrics-hcp-5b868d7ddd-fhcvs\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd16iscun46vl67eantdmfbrchq5cvk-q4u5f1h8q8y4t5a/kube-state-metrics-hcp-5b868d7ddd-fhcvs has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-24T15:33:32Z\n Ended: 2026-08-24T15:39:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1362816-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd15s2tbl2mk3taqpkr4l2ue96f7dbj-agg-cluster\", pod=\"control-plane-operator-6dffd99944-b2sdg\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd15s2tbl2mk3taqpkr4l2ue96f7dbj-agg-cluster/control-plane-operator-6dffd99944-b2sdg has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"affaeca72c47255b1e792ce7fbdd676958c87bc90e8b0ad590a362a1f1d2efd1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","failed_at":"e2e","signature_id":"14a501f15a848fe949827032cbe7eab1732753bec28bb5cacc0d3876ab277f28","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001580bc0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-cbbn9kpl7fxv\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c6f112da-48e3-4cbb-a1b9-a47159e85ff5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c6f112da-48e3-4cbb-a1b9-a47159e85ff5\",\n \"name\": \"c6f112da-48e3-4cbb-a1b9-a47159e85ff5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:18:01.504032647Z\",\n \"endTime\": \"2026-08-24T15:37:08.017891025Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, azure-cloud-controller-manager, cluster-node-tuning-operator, olm-operator, olm-collect-profiles, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, community-operators-catalog, redhat-operators-catalog, machine-approver, openshift-controller-manager, cluster-storage-operator, dns-operator, redhat-marketplace-catalog, cluster-version-operator, openshift-route-controller-manager, ingress-operator, kube-scheduler, cluster-image-registry-operator, konnectivity-agent, catalog-operator, cluster-network-operator, ignition-server, ignition-server-proxy, openshift-apiserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001580bc0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-cbbn9kpl7fxv\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c6f112da-48e3-4cbb-a1b9-a47159e85ff5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c6f112da-48e3-4cbb-a1b9-a47159e85ff5\",\n \"name\": \"c6f112da-48e3-4cbb-a1b9-a47159e85ff5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:18:01.504032647Z\",\n \"endTime\": \"2026-08-24T15:37:08.017891025Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, azure-cloud-controller-manager, cluster-node-tuning-operator, olm-operator, olm-collect-profiles, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, community-operators-catalog, redhat-operators-catalog, machine-approver, openshift-controller-manager, cluster-storage-operator, dns-operator, redhat-marketplace-catalog, cluster-version-operator, openshift-route-controller-manager, ingress-operator, kube-scheduler, cluster-image-registry-operator, konnectivity-agent, catalog-operator, cluster-network-operator, ignition-server, ignition-server-proxy, openshift-apiserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b8a33e9d98ee8d02e1542a6636d65c44432c75ceb4ba2640fcc9e74593e6aca1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","failed_at":"e2e","signature_id":"f45d8f4fc578e9bcbc89c15b0ea557b08d4323587837d33c561c269a0b539be9","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e96560\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-fn2m5nmfttf7\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\",\n \"name\": \"eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:17:08.176674236Z\",\n \"endTime\": \"2026-08-24T15:36:17.96458443Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e96560\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-fn2m5nmfttf7\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\",\n \"name\": \"eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:17:08.176674236Z\",\n \"endTime\": \"2026-08-24T15:36:17.96458443Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"daba2526e5b426f2d9b8035353c0b6d30ea4645700dbc811693b5ac725c5a0ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","failed_at":"e2e","signature_id":"113a28e0e41ddb16e2d1b2e27f386f97c1c16af7bfd0bd1391cd74394fd64a44","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--wkl8qfr9l4cs/cluster-hs-pre-jtp7jx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009c0960\u003e: \n failed to create HCP cluster cluster-hs-pre-jtp7jx: failed waiting for cluster=\"cluster-hs-pre-jtp7jx\" in resourcegroup=\"rg-hs-pre--wkl8qfr9l4cs\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\",\n \"name\": \"df253c01-fd54-4465-9725-47d0942d268e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:18:48.695053949Z\",\n \"endTime\": \"2026-08-24T15:37:57.993163664Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--wkl8qfr9l4cs/cluster-hs-pre-jtp7jx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009c0960\u003e: \n failed to create HCP cluster cluster-hs-pre-jtp7jx: failed waiting for cluster=\"cluster-hs-pre-jtp7jx\" in resourcegroup=\"rg-hs-pre--wkl8qfr9l4cs\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\",\n \"name\": \"df253c01-fd54-4465-9725-47d0942d268e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:18:48.695053949Z\",\n \"endTime\": \"2026-08-24T15:37:57.993163664Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c3db9c7d221b5f14f4cb052b432def48559d4c9a3619db1db26cc57a99142ed9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dccaeed8479de8729ea834a46f2448666c4518fb3b7e49eac112e504be3435eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","failed_at":"e2e","signature_id":"29b0688020a02097d8409be35d56bc8e657d156f249e7e0111a076d22256e797","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:599]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e96960\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-q2tthc\" in resourcegroup=\"rg-np-downgrade-q2tthc-z7rbl7r2jwws\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47831462-3285-4a15-8722-8601b5708cf5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47831462-3285-4a15-8722-8601b5708cf5\",\n \"name\": \"47831462-3285-4a15-8722-8601b5708cf5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:29:26.286804415Z\",\n \"endTime\": \"2026-08-24T15:48:34.685207308Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:599]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e96960\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-q2tthc\" in resourcegroup=\"rg-np-downgrade-q2tthc-z7rbl7r2jwws\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47831462-3285-4a15-8722-8601b5708cf5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47831462-3285-4a15-8722-8601b5708cf5\",\n \"name\": \"47831462-3285-4a15-8722-8601b5708cf5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:29:26.286804415Z\",\n \"endTime\": \"2026-08-24T15:48:34.685207308Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","c3db9c7d221b5f14f4cb052b432def48559d4c9a3619db1db26cc57a99142ed9","e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895787960143872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"0633420cc119b5143a145776d79b9bd204c16df0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T14:29:42Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e717807aab748d648493fa2186de75d7044c4f28313d1a2389bbb5aa3fdaa289","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895787960143872","occurred_at":"2026-08-24T14:29:42Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6658,"pr_state":"open","pr_sha":"9906d38df8dc349a74b8507e4059d71747a87617","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T14:28:36Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"54e71ffcc90c125aaeae0df0f9f11ebf0753af04b2bc2456f7606ba9b3ee0a99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","failed_at":"e2e","signature_id":"ccd1c15e8b2737a8cbb30d07ada8f1b6f0748afd9d43b748ec4128f7a61ce044","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008f0180\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fvnthdb9mrqx\" to finish creating: GET https://rp.j2797056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3adf48b6-e1aa-4b6a-93ab-0fddaf491e0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3adf48b6-e1aa-4b6a-93ab-0fddaf491e0c\",\n \"name\": \"3adf48b6-e1aa-4b6a-93ab-0fddaf491e0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:18:10.302856594Z\",\n \"endTime\": \"2026-08-24T15:37:11.239396006Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008f0180\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fvnthdb9mrqx\" to finish creating: GET https://rp.j2797056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3adf48b6-e1aa-4b6a-93ab-0fddaf491e0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3adf48b6-e1aa-4b6a-93ab-0fddaf491e0c\",\n \"name\": \"3adf48b6-e1aa-4b6a-93ab-0fddaf491e0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:18:10.302856594Z\",\n \"endTime\": \"2026-08-24T15:37:11.239396006Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7bcc9144ca44870d3315f83513c9a9a4e6c8783464620b0ebf4b5a5d58ccadcb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"94dbefdb7d88f380bcc7f74ab9b59546aa323024837e4c23701a94f7c4d63455","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","failed_at":"e2e","signature_id":"2cc87e67c265ba4f74a9a313e8f38a6f54437455cae5243f5e1f2cc3556cdfc9","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001078580\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-96jvrlw87dhc\" to finish creating: GET https://rp.j2797056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f52090fa-71d3-447f-ad30-f731526cbb4b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f52090fa-71d3-447f-ad30-f731526cbb4b\",\n \"name\": \"f52090fa-71d3-447f-ad30-f731526cbb4b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:16:10.209370894Z\",\n \"endTime\": \"2026-08-24T15:35:11.305317842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001078580\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-96jvrlw87dhc\" to finish creating: GET https://rp.j2797056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f52090fa-71d3-447f-ad30-f731526cbb4b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f52090fa-71d3-447f-ad30-f731526cbb4b\",\n \"name\": \"f52090fa-71d3-447f-ad30-f731526cbb4b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:16:10.209370894Z\",\n \"endTime\": \"2026-08-24T15:35:11.305317842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ead866abad63084957ffceb86c3a940a2b5679862f356e6f2839ef2588e43a9b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","failed_at":"e2e","signature_id":"f749c71c85c572aa2c57394be8a931e71c30eda717753e81fa6456d997c9c5fc","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000698540\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-rqwql22jhw9b\" to finish creating: GET https://rp.j2797056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9120d0f9-847c-44de-98bf-37d19b45ef2f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9120d0f9-847c-44de-98bf-37d19b45ef2f\",\n \"name\": \"9120d0f9-847c-44de-98bf-37d19b45ef2f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:16:29.566422332Z\",\n \"endTime\": \"2026-08-24T15:35:31.366369174Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000698540\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-rqwql22jhw9b\" to finish creating: GET https://rp.j2797056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9120d0f9-847c-44de-98bf-37d19b45ef2f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9120d0f9-847c-44de-98bf-37d19b45ef2f\",\n \"name\": \"9120d0f9-847c-44de-98bf-37d19b45ef2f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:16:29.566422332Z\",\n \"endTime\": \"2026-08-24T15:35:31.366369174Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6232,"pr_state":"open","pr_sha":"cbb67883add3cc85dd65c43e81a6cb584dbf8021","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T14:18:07Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"39bee5c62fefc6167ba6c2d5d0beee090d3dbadec497bacb5e08cbb0d8a81017","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","failed_at":"e2e","signature_id":"7df46629584e4c4253c91624b483f13865f970598c9ed547ff855ac9eea68fe2","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0010deff0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-4cmmkc\" in resourcegroup=\"admin-credential-lifecycle-test-mk5bt6ghv29k\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\",\n \"name\": \"0644f278-df05-4603-a10d-3a66a1f851d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:17:03.672410989Z\",\n \"endTime\": \"2026-08-24T15:41:10.046088761Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tjfmep0gog9fd4pfkr5bnpc0n642 still exists (deletion dispatched at 2026-08-24T15:17:04Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0010deff0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-4cmmkc\" in resourcegroup=\"admin-credential-lifecycle-test-mk5bt6ghv29k\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\",\n \"name\": \"0644f278-df05-4603-a10d-3a66a1f851d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:17:03.672410989Z\",\n \"endTime\": \"2026-08-24T15:41:10.046088761Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tjfmep0gog9fd4pfkr5bnpc0n642 still exists (deletion dispatched at 2026-08-24T15:17:04Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"991b62bbe6cfc8a693325e917df0812a382fcf8c546cb468c5f57a28f872fbbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","failed_at":"e2e","signature_id":"cf9ddb524cae2c0c6a37a93000fda2cd14fe56bff50daec6b5b3ed77ad21467c","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001c982a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-gdnhnw\" in resourcegroup=\"rg-hs-pre--hm7prkdxbkw6\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\",\n \"name\": \"67ed1acc-0077-4b07-848d-484f46170f44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:21:50.42303027Z\",\n \"endTime\": \"2026-08-24T15:45:59.872804145Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001c982a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-gdnhnw\" in resourcegroup=\"rg-hs-pre--hm7prkdxbkw6\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\",\n \"name\": \"67ed1acc-0077-4b07-848d-484f46170f44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:21:50.42303027Z\",\n \"endTime\": \"2026-08-24T15:45:59.872804145Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bad17b00fd2861c199fb8c7332ad5eb58ff1eb35d64efc7066c892df3da78afa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] \u003cno_message\u003e; [clusterServiceStatus] \u003cno_message\u003e; [descendantResources] \u003cno_message\u003e; [hostedCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d9eac3c70b13798d35ede839155937dec632af2fc452cedfaed5c4c12a98908f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","failed_at":"e2e","signature_id":"53ff086f26d6ee86f70c417c33372b3bc3883205891aacb5f42319d457fb9cf5","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000e8af48\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mb2sm98hvbm8\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n \"name\": \"76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:14:43.826676557Z\",\n \"endTime\": \"2026-08-24T15:38:50.051171369Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm25j05l3q2f8a1e83cjmn4lda8g still exists (deletion dispatched at 2026-08-24T15:14:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000e8af48\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mb2sm98hvbm8\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n \"name\": \"76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:14:43.826676557Z\",\n \"endTime\": \"2026-08-24T15:38:50.051171369Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm25j05l3q2f8a1e83cjmn4lda8g still exists (deletion dispatched at 2026-08-24T15:14:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f247ffe841ae32ab6f0a2ca9f5a14d616afb290a6a52bd6bd5aad6bf0a8b0d93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","failed_at":"e2e","signature_id":"60c407c2c17844ded754b0d5e8dda40f125391008bbbdd669c23fa4348b5b316","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000ebc2a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-h4ldj6\" in resourcegroup=\"rg-candidate-5-0-xjnldggtrt7m\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\",\n \"name\": \"fa0102ea-068c-4907-92f2-69579c623cac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:21:51.698922375Z\",\n \"endTime\": \"2026-08-24T15:45:59.987785659Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm5ibpilvjmdnr77hbqf9t255bon still exists (deletion dispatched at 2026-08-24T15:21:51Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000ebc2a0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-h4ldj6\" in resourcegroup=\"rg-candidate-5-0-xjnldggtrt7m\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\",\n \"name\": \"fa0102ea-068c-4907-92f2-69579c623cac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T15:21:51.698922375Z\",\n \"endTime\": \"2026-08-24T15:45:59.987785659Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm5ibpilvjmdnr77hbqf9t255bon still exists (deletion dispatched at 2026-08-24T15:21:51Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ba89a4166305f6c19c8eaadc8056c4375814afedeefcd4ac9c286060c7d5cad1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fee1813916a1d5e8ada0b263bfbafaa4e599bbee28edb7e6b8c5c2ac543d2db8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","failed_at":"alert","signature_id":"d55af07fa2830b9ad7b195c673e4e375ec2e39f3e870f6e6e4139ca2af8abfa0","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s) (5 unknown, 7 known)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T15:11:48Z\n Ended: 2026-08-24T15:18:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9012096-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd0tivj0u60lm1dth5pjmkuarro34do-n8n9o9a8t2s4e6m\", pod=\"kube-apiserver-795c5dc94f-bggd8\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd0tivj0u60lm1dth5pjmkuarro34do-n8n9o9a8t2s4e6m/kube-apiserver-795c5dc94f-bggd8 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-08-24T15:11:48Z\n Ended: 2026-08-24T15:18:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9012096-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd0tsdt2doqqre7oevhvs2m0rcviir2-w9k0g8e9q6b2y5a\", pod=\"etcd-1\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd0tsdt2doqqre7oevhvs2m0rcviir2-w9k0g8e9q6b2y5a/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-08-24T15:11:48Z\n Ended: 2026-08-24T15:18:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9012096-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd0tl62srf8l23gsr4t3od82f106gap-m2c3u3c1r4o8d7i\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd0tl62srf8l23gsr4t3od82f106gap-m2c3u3c1r4o8d7i/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-08-24T15:11:48Z\n Ended: 2026-08-24T15:18:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9012096-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd0tcss6f44egtai516gd8nov4k698p-v20260901\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd0tcss6f44egtai516gd8nov4k698p-v20260901/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-08-24T15:11:48Z\n Ended: 2026-08-24T15:18:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9012096-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd0tlmtcpd07kvkm18o0djt9sfj7dcm-n3g9j2h5i2r5y3t\", pod=\"etcd-2\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sd0tlmtcpd07kvkm18o0djt9sfj7dcm-n3g9j2h5i2r5y3t/etcd-2 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","ba89a4166305f6c19c8eaadc8056c4375814afedeefcd4ac9c286060c7d5cad1","bad17b00fd2861c199fb8c7332ad5eb58ff1eb35d64efc7066c892df3da78afa"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091887496571916288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"0633420cc119b5143a145776d79b9bd204c16df0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T13:56:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c0edeaf3f8395af925f8cfbc263af86fdade74c7453b01e4d458a211c3bbc96b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091887496571916288","occurred_at":"2026-08-24T13:56:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091887378032496640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6645,"pr_state":"open","pr_sha":"3c81c748e92c32ebefa3e5019d3bd3f86cf04336","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T13:56:30Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d23c0d738ff953325181ff67ef6c73246a2364c8f4f2764bcbfa50648bd9cad9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091887378032496640","occurred_at":"2026-08-24T13:56:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091880353525403648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6585,"pr_state":"closed","pr_sha":"49b477321c95eb9cd733e0e2e4a2c52dc5e96d79","final_merged_sha":"49b477321c95eb9cd733e0e2e4a2c52dc5e96d79","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-24T13:28:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879964126220288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6645,"pr_state":"open","pr_sha":"3c81c748e92c32ebefa3e5019d3bd3f86cf04336","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T13:27:03Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e5a752e104974cb1a1c176cda10d20c0c326a9e7bf3a5a0b6ffe40fb07d0093f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879964126220288","occurred_at":"2026-08-24T13:27:03Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879954055696384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6429,"pr_state":"open","pr_sha":"d31f1425ab99ac4190688793ddf767c13e257968","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T13:27:00Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3e0cfa6e25ee886928f83666112903cd95a058fdfc908ddcc3fed9b668274c61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879954055696384","occurred_at":"2026-08-24T13:27:00Z","failed_at":"e2e","signature_id":"d079461036286dee8f18df87fbaabb5b38f1fcaa64aea1bbd3a2fe3d558a9ddb","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:160]: kubectl version should succeed from VM via private KAS internal LB (output: )\nUnexpected error:\n \u003c*errors.errorString | 0xc0007fbf60\u003e: \n run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:160]: kubectl version should succeed from VM via private KAS internal LB (output: )\nUnexpected error:\n \u003c*errors.errorString | 0xc0007fbf60\u003e: \n run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5d433008aba23c344781b86d474b8dbf122c3c1ce0be7c5095e73ef5b595140f","failure_pattern":"run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"\u003cvm\u003e\" (output: \"Client Version: v1.36.4 Kustomize Version: v5.8.1\")","search_query":"run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")"}},{"row_id":"4ecd65745e65385d36e7f225fd065b11f3fa5b9131f88668d53a86aa49df9d38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879954055696384","occurred_at":"2026-08-24T13:27:00Z","failed_at":"e2e","signature_id":"b66537b24ffd588d399e5e7cc810bf3a188e5e0dce669b0b16956a016a18a10c","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--k24nwww6fmhh/cluster-hs-pre-zdf4g7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111c160\u003e: \n failed to create HCP cluster cluster-hs-pre-zdf4g7: failed waiting for cluster=\"cluster-hs-pre-zdf4g7\" in resourcegroup=\"rg-hs-pre--k24nwww6fmhh\" to finish creating: GET https://rp.j5696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0975238-90c3-4b41-87eb-c874aa41d8d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0975238-90c3-4b41-87eb-c874aa41d8d7\",\n \"name\": \"a0975238-90c3-4b41-87eb-c874aa41d8d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T14:14:48.351398791Z\",\n \"endTime\": \"2026-08-24T14:33:48.407943371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--k24nwww6fmhh/cluster-hs-pre-zdf4g7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111c160\u003e: \n failed to create HCP cluster cluster-hs-pre-zdf4g7: failed waiting for cluster=\"cluster-hs-pre-zdf4g7\" in resourcegroup=\"rg-hs-pre--k24nwww6fmhh\" to finish creating: GET https://rp.j5696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0975238-90c3-4b41-87eb-c874aa41d8d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0975238-90c3-4b41-87eb-c874aa41d8d7\",\n \"name\": \"a0975238-90c3-4b41-87eb-c874aa41d8d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T14:14:48.351398791Z\",\n \"endTime\": \"2026-08-24T14:33:48.407943371Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","5d433008aba23c344781b86d474b8dbf122c3c1ce0be7c5095e73ef5b595140f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879927736438784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6433,"pr_state":"open","pr_sha":"9ec65054f149d1b7507171a2d613008ef1bc9761","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T13:26:52Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0d31df0fe4c66f681f8190eb9e57b4fe742175e2e29d828856ce74e2e61b55f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879927736438784","occurred_at":"2026-08-24T13:26:52Z","failed_at":"e2e","signature_id":"6a060f99c27004cf01810556d4b6e1126f6aeee13d0cc7b3c046327b6e9bcc0d","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f24820\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jdqbgdfs67t9\" to finish creating: GET https://rp.j6438784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/733891b5-0d3d-4dee-bbf2-0fd76696850b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/733891b5-0d3d-4dee-bbf2-0fd76696850b\",\n \"name\": \"733891b5-0d3d-4dee-bbf2-0fd76696850b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T14:17:00.887821869Z\",\n \"endTime\": \"2026-08-24T14:36:05.25428442Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f24820\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jdqbgdfs67t9\" to finish creating: GET https://rp.j6438784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/733891b5-0d3d-4dee-bbf2-0fd76696850b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/733891b5-0d3d-4dee-bbf2-0fd76696850b\",\n \"name\": \"733891b5-0d3d-4dee-bbf2-0fd76696850b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T14:17:00.887821869Z\",\n \"endTime\": \"2026-08-24T14:36:05.25428442Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d64c4e422b2f3ce74e6e3724d780ac11ef4226fba01dd56602e3eed32c025246","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879927736438784","occurred_at":"2026-08-24T13:26:52Z","failed_at":"alert","signature_id":"f42ba6cef1b20012612134284fc91db6cdee01f2a97291f342457df1df0028e6","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-08-24T14:40:10Z\n Ended: 2026-08-24T14:49:11Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6438784-svc\", component=\"backend\", name=\"csstatedump\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091873554273406976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6643,"pr_state":"open","pr_sha":"4cf6086e8b2aa130d65c27d7b6e7fb155bbdab8b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T13:01:35Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ea88ab83bd22caff4ec7ab44b27138647e7249533c86cdb8a9087115a86e0503","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091873554273406976","occurred_at":"2026-08-24T13:01:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091873075288084480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6618,"pr_state":"open","pr_sha":"669f42e493665b25684e55d7e521b8c169ca1f38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T12:59:40Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"24cc3f01b4a5c3d6b0b51a7dd6f3f056bfabd9939bf1e3d051027a9f2f15a4ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091873075288084480","occurred_at":"2026-08-24T12:59:40Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T13:26:56.540Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:26:58.762Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T13:27:00.485Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=abeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8084480-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fabeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969\ntime=2026-08-24T13:35:32.482Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/abeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969/operationStatuses/08584140288659502671\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/cluster-kyv757fphd2fm\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j8084480-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-24652102-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-24652102-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872721045557248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T12:58:16Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"075d8d816ed885b5524d2169d59d3cbf24c2156d514210d5f53490700badd474","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872721045557248","occurred_at":"2026-08-24T12:58:16Z","failed_at":"provision","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-08-24T13:31:41.774Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:31:42.863Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T13:31:44.020Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=cf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5557248-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5\ntime=2026-08-24T13:39:15.842Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/cf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5/operationStatuses/08584140285820210806\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/cluster-ce5kkcjcnxbu2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5557248-mgmt-2/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-20013346-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-20013346-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872693082132480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"0633420cc119b5143a145776d79b9bd204c16df0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T12:58:09Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9ad130438d9164020299c3baeccc25a22f45d40dfb1baf09903424f257361caa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872693082132480","occurred_at":"2026-08-24T12:58:09Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T13:26:28.921Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:26:30.957Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T13:26:32.602Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2132480-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848\ntime=2026-08-24T13:34:04.585Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848/operationStatuses/08584140288938416315\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/cluster-d54rkmqgu5xlq\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2132480-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-42034125-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-42034125-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872666397970432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6504,"pr_state":"open","pr_sha":"af0ab1f037eef8141b97aadd390cc54296b6d1c3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T12:58:03Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c11c1fae00c9c1336a91bc279e28dcd236833d1d1789682645e3a3de1fd48d9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872666397970432","occurred_at":"2026-08-24T12:58:03Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-08-24T13:31:05.820Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:31:07.064Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-08-24T13:31:08.065Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=ed1457ef8895351af8e0cf66909d3e8cde536aaa7acb6606665b7650f6a24978 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7970432-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fed1457ef8895351af8e0cf66909d3e8cde536aaa7acb6606665b7650f6a24978\ntime=2026-08-24T13:39:10.218Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7970432-mgmt-1/providers/Microsoft.Resources/deployments/ed1457ef8895351af8e0cf66909d3e8cde536aaa7acb6606665b7650f6a24978/operationStatuses/08584140286179722113\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7970432-mgmt-1/providers/Microsoft.Resources/deployments/cluster-bydjouzz4qrms\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7970432-mgmt-1/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7970432-mgmt-1/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7970432-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7970432-mgmt-1/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7970432-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-20554033-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7970432-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-20554033-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091868107806085120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6585,"pr_state":"closed","pr_sha":"49b477321c95eb9cd733e0e2e4a2c52dc5e96d79","final_merged_sha":"49b477321c95eb9cd733e0e2e4a2c52dc5e96d79","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-24T12:39:55Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9f7490c7e7677fc384e63ed5db58d8abd0aca64db81aa21c243b18f53388e9ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091868107806085120","occurred_at":"2026-08-24T12:39:55Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T13:06:20.681Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:06:22.594Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T13:06:24.444Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=03ed80b7fb9f53548d372e1053a4aca53aea99bb193680d269ceba2e4aa413ec portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6085120-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F03ed80b7fb9f53548d372e1053a4aca53aea99bb193680d269ceba2e4aa413ec\ntime=2026-08-24T13:14:56.464Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6085120-svc/providers/Microsoft.Resources/deployments/03ed80b7fb9f53548d372e1053a4aca53aea99bb193680d269ceba2e4aa413ec/operationStatuses/08584140301019888549\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6085120-svc/providers/Microsoft.Resources/deployments/cluster-mw4vmkbkun3py\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6085120-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6085120-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6085120-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j6085120-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6085120-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-45523988-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6085120-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-45523988-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091867251824136192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6585,"pr_state":"closed","pr_sha":"49b477321c95eb9cd733e0e2e4a2c52dc5e96d79","final_merged_sha":"49b477321c95eb9cd733e0e2e4a2c52dc5e96d79","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-24T12:36:32Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d1d8a25a7919ec61a48b4748839e443e4dcfacb6c49d8900cb4869f2dce1a53d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091867251824136192","occurred_at":"2026-08-24T12:36:32Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-08-24T13:04:34.083Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:04:35.141Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-08-24T13:04:36.377Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=4989b67b02de49f77c8217e80b11221be6396694deee2c6d1f0f2599c72fa63b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4136192-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4989b67b02de49f77c8217e80b11221be6396694deee2c6d1f0f2599c72fa63b\ntime=2026-08-24T13:12:38.165Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4136192-mgmt-1/providers/Microsoft.Resources/deployments/4989b67b02de49f77c8217e80b11221be6396694deee2c6d1f0f2599c72fa63b/operationStatuses/08584140302097177966\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4136192-mgmt-1/providers/Microsoft.Resources/deployments/cluster-n72exdzvr6uha\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4136192-mgmt-1/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4136192-mgmt-1/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4136192-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j4136192-mgmt-1/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4136192-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-93565555-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4136192-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-93565555-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091856057667162112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6585,"pr_state":"closed","pr_sha":"49b477321c95eb9cd733e0e2e4a2c52dc5e96d79","final_merged_sha":"49b477321c95eb9cd733e0e2e4a2c52dc5e96d79","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-24T11:52:03Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4f3afe345cac29b3c107ae6e6389ffd24db428b0c8b68b495d8ed008c7cade1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091856057667162112","occurred_at":"2026-08-24T11:52:03Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T12:17:22.416Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T12:17:24.567Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T12:17:26.247Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=b81d2845865e09100d21919fab3e6ce78d71d88f3ac75e34d8fcd2a7b7e34e97 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7162112-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb81d2845865e09100d21919fab3e6ce78d71d88f3ac75e34d8fcd2a7b7e34e97\ntime=2026-08-24T12:25:28.244Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7162112-svc/providers/Microsoft.Resources/deployments/b81d2845865e09100d21919fab3e6ce78d71d88f3ac75e34d8fcd2a7b7e34e97/operationStatuses/08584140330401352534\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7162112-svc/providers/Microsoft.Resources/deployments/cluster-fjywew6dvq3ek\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7162112-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7162112-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7162112-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j7162112-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7162112-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-30429984-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7162112-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-30429984-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6070,"pr_state":"open","pr_sha":"2f68bd3b981a9e093b933e26b060f8e1150c0eba","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T11:48:53Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"43850b202ca0ff22e3a55c75d958242eb90330d0c040558210f71104d62e5efd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","failed_at":"e2e","signature_id":"25baffda5fc6f88c385cae06e401a6f4815895c798a9d1e087a18364db777535","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001600420\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-rhjcsskbzqd7\" to finish creating: GET https://rp.j3221632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fdd8d463-3794-4ddc-a4f8-6d580bc2c2a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fdd8d463-3794-4ddc-a4f8-6d580bc2c2a9\",\n \"name\": \"fdd8d463-3794-4ddc-a4f8-6d580bc2c2a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T12:34:25.475476952Z\",\n \"endTime\": \"2026-08-24T12:53:28.435563679Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001600420\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-rhjcsskbzqd7\" to finish creating: GET https://rp.j3221632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fdd8d463-3794-4ddc-a4f8-6d580bc2c2a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fdd8d463-3794-4ddc-a4f8-6d580bc2c2a9\",\n \"name\": \"fdd8d463-3794-4ddc-a4f8-6d580bc2c2a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T12:34:25.475476952Z\",\n \"endTime\": \"2026-08-24T12:53:28.435563679Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"625ddf45cadf3332a322cff042972690aa6d87b800c44141c7bac9d49046fd3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","failed_at":"e2e","signature_id":"78869f45f7134e749985fc5b23dff71b456538a41d70734a716f1e608670ab6c","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00132c390\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00132c390\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d3766fe34f03f22876c777e51a7be1734ee728533bfb3f28e111bf0b35fcad1b","failure_pattern":"UpdateHCPCluster20260630 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded"}},{"row_id":"daebb88c61ecc4072b36b1d63e2f174749875655d91f4362abff9051a3148e1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","failed_at":"e2e","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fac505f52469073fa6a64250c64e44813d69018a355acebf1fdde81a505c53f0","failure_pattern":"expected 3 nodes to have original label key1=value1","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1"}},{"row_id":"e7442581dc7f7333d318a945ff659a474fa87c012823e9bc2d5d88b0958efbf8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","failed_at":"e2e","signature_id":"dfb206f2ba6bc7bc3ce13c32e974065647bae3407bafdce87be1e26f94b6936e","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00396d0e0\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00396d0e0\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","d3766fe34f03f22876c777e51a7be1734ee728533bfb3f28e111bf0b35fcad1b","fac505f52469073fa6a64250c64e44813d69018a355acebf1fdde81a505c53f0"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091852685220253696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6424,"pr_state":"open","pr_sha":"d1ddfa5f639415ec37f6bedcd914d082e895d8ad","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T11:34:18Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"499027f5b3b6c8b5d6086f7afd02d0a04174338848a6a4f08b01000c4d53b940","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091852685220253696","occurred_at":"2026-08-24T11:34:18Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T12:10:15.954Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T12:10:17.764Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T12:10:19.676Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=4fb85e22a773a3391f4c0348e0b5470c084630ac275bea53bc5bf265bbbb61e3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0253696-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4fb85e22a773a3391f4c0348e0b5470c084630ac275bea53bc5bf265bbbb61e3\ntime=2026-08-24T12:18:21.562Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0253696-svc/providers/Microsoft.Resources/deployments/4fb85e22a773a3391f4c0348e0b5470c084630ac275bea53bc5bf265bbbb61e3/operationStatuses/08584140334670745974\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0253696-svc/providers/Microsoft.Resources/deployments/cluster-tswqsaf7g3tae\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0253696-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0253696-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0253696-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j0253696-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0253696-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-37903298-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0253696-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-37903298-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844440330080256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T11:05:53Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"715c12e3ee8d180d8877ef26a8b4fd621f15ad4c5c882b7ac88c0d96e4396d9b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844440330080256","occurred_at":"2026-08-24T11:05:53Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T11:34:04.464Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T11:34:07.135Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T11:34:09.248Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=9b211860af6b0c145b57f6330b1e13702971f10380c81f1e4f30ef18196d35a3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0080256-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9b211860af6b0c145b57f6330b1e13702971f10380c81f1e4f30ef18196d35a3\ntime=2026-08-24T11:43:11.648Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0080256-svc/providers/Microsoft.Resources/deployments/9b211860af6b0c145b57f6330b1e13702971f10380c81f1e4f30ef18196d35a3/operationStatuses/08584140356374348833\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0080256-svc/providers/Microsoft.Resources/deployments/cluster-dkoaaglt4pes6\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0080256-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0080256-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0080256-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j0080256-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0080256-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-10945567-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0080256-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-10945567-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0080256-svc/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0080256-svc/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0080256-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j0080256-svc/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0080256-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-17123608-vmss'. Please try alternative sizes 'Standard_D8ds_v6' (zones 3, 2), 'Standard_D4ads_v7' (zones 3, 1, 2), 'Standard_D4s_v6' (zones 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0080256-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-17123608-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844377495212032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"0633420cc119b5143a145776d79b9bd204c16df0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T11:05:38Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3d1b1e670ac496fb553fc7c30adcab76d44f911cab1d45b9f38eed7bcc6f4a8f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844377495212032","occurred_at":"2026-08-24T11:05:38Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T11:32:20.643Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T11:32:23.051Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T11:32:24.783Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=74ef5b6bb1b5c6fd3ba7cf88a598fe1556c944274c8643c2c60a526c2eccdb5a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5212032-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F74ef5b6bb1b5c6fd3ba7cf88a598fe1556c944274c8643c2c60a526c2eccdb5a\ntime=2026-08-24T11:41:26.957Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5212032-svc/providers/Microsoft.Resources/deployments/74ef5b6bb1b5c6fd3ba7cf88a598fe1556c944274c8643c2c60a526c2eccdb5a/operationStatuses/08584140357416951822\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5212032-svc/providers/Microsoft.Resources/deployments/cluster-3hrc2fvonf4ms\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5212032-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5212032-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5212032-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j5212032-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5212032-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-38464807-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5212032-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-38464807-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844284889174016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6504,"pr_state":"open","pr_sha":"af0ab1f037eef8141b97aadd390cc54296b6d1c3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T11:05:16Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"01c33f825dd1aae7b32b57883b9b0c752691ccab1985b37b40cabc025cbc9bfc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844284889174016","occurred_at":"2026-08-24T11:05:16Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-08-24T11:29:35.754Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T11:29:37.122Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-08-24T11:29:38.565Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=06d9da801c490b565266bf3066adb6655831ea121ede6c8b88ef3955cc91c2af portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9174016-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F06d9da801c490b565266bf3066adb6655831ea121ede6c8b88ef3955cc91c2af\ntime=2026-08-24T11:38:10.769Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9174016-mgmt-1/providers/Microsoft.Resources/deployments/06d9da801c490b565266bf3066adb6655831ea121ede6c8b88ef3955cc91c2af/operationStatuses/08584140359078639543\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9174016-mgmt-1/providers/Microsoft.Resources/deployments/cluster-tztc6ggh322ps\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9174016-mgmt-1/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9174016-mgmt-1/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9174016-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j9174016-mgmt-1/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9174016-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-11156693-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9174016-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-11156693-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091840235057975296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6618,"pr_state":"open","pr_sha":"669f42e493665b25684e55d7e521b8c169ca1f38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T10:49:11Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"55bbdc10d1d4208ceb032552fc971bdcce7a3b21519592436ef08b35312ad264","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091840235057975296","occurred_at":"2026-08-24T10:49:11Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T10:57:28.271Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T10:57:30.601Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T10:57:32.308Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=000c9f6cacb3aef7ed0c72330601e9df4ede8a902eb59bd2ba5f467f93427386 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7975296-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F000c9f6cacb3aef7ed0c72330601e9df4ede8a902eb59bd2ba5f467f93427386\ntime=2026-08-24T11:12:35.760Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7975296-svc/providers/Microsoft.Resources/deployments/000c9f6cacb3aef7ed0c72330601e9df4ede8a902eb59bd2ba5f467f93427386/operationStatuses/08584140378341181394\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7975296-svc/providers/Microsoft.Resources/deployments/cluster-gxrvf2fjg2hfw\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7975296-svc/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7975296-svc/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7975296-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j7975296-svc/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7975296-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-96441150-vmss'. Please try alternative sizes 'Standard_D8ds_v6' (zones 2, 3), 'Standard_D4ads_v7' (zones 2, 1, 3), 'Standard_D4s_v6' (zones 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7975296-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-96441150-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7975296-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7975296-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7975296-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j7975296-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7975296-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-57198211-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7975296-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-57198211-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6656/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091833425257304064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6656,"pr_state":"closed","pr_sha":"8d7f552861d56e8b6118f9edd6a7f5a6059ddf76","final_merged_sha":"8d7f552861d56e8b6118f9edd6a7f5a6059ddf76","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-24T10:22:07Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091827867787726848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6070,"pr_state":"open","pr_sha":"d33b7bca55222fc9cc5b82d9e112e29c5b823f11","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T10:00:02Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d7eeff44130eba85a75ddb85f0ae090cbfa20e40c1b954f0267ded85b23d7f23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091827867787726848","occurred_at":"2026-08-24T10:00:02Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-08-24T10:24:43.889Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T10:24:45.005Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-08-24T10:24:46.196Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=b901b6ce5731a24b329db131b89a909d69b5433ca33c57bfdd39a838d9879a8e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7726848-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb901b6ce5731a24b329db131b89a909d69b5433ca33c57bfdd39a838d9879a8e\ntime=2026-08-24T10:33:18.137Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7726848-mgmt-1/providers/Microsoft.Resources/deployments/b901b6ce5731a24b329db131b89a909d69b5433ca33c57bfdd39a838d9879a8e/operationStatuses/08584140397998744039\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7726848-mgmt-1/providers/Microsoft.Resources/deployments/cluster-cnl6ie2kiux2u\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7726848-mgmt-1/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7726848-mgmt-1/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7726848-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7726848-mgmt-1/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7726848-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-15008703-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7726848-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-15008703-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091827277728845824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6429,"pr_state":"open","pr_sha":"d31f1425ab99ac4190688793ddf767c13e257968","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T09:57:41Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"89de4f6f4be0b50268e3f9a0b0d21980975137bf48ac458e04965fd40f17e035","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091827277728845824","occurred_at":"2026-08-24T09:57:41Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T10:07:30.028Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T10:07:32.346Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T10:07:34.256Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=d1f5df56795df1062698fb6ccb81ccf8210e23e07169776ac2ac1208670e13c1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8845824-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd1f5df56795df1062698fb6ccb81ccf8210e23e07169776ac2ac1208670e13c1\ntime=2026-08-24T10:17:06.305Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8845824-svc/providers/Microsoft.Resources/deployments/d1f5df56795df1062698fb6ccb81ccf8210e23e07169776ac2ac1208670e13c1/operationStatuses/08584140408323370960\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8845824-svc/providers/Microsoft.Resources/deployments/cluster-ojjgxet7r56au\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8845824-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8845824-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8845824-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j8845824-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8845824-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-31090874-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8845824-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-31090874-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091823405572558848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6433,"pr_state":"open","pr_sha":"9ec65054f149d1b7507171a2d613008ef1bc9761","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T09:42:18Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"101f079f53a49d1a138c5971f0e7da300a930206c324fe4f959cf079da57dc49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091823405572558848","occurred_at":"2026-08-24T09:42:18Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T10:09:53.878Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T10:09:55.657Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T10:09:57.313Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=70eedf005776c816ac9aa5070bf370704152d4348d77e746226f8f6d195d2b70 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2558848-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F70eedf005776c816ac9aa5070bf370704152d4348d77e746226f8f6d195d2b70\ntime=2026-08-24T10:17:59.005Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2558848-svc/providers/Microsoft.Resources/deployments/70eedf005776c816ac9aa5070bf370704152d4348d77e746226f8f6d195d2b70/operationStatuses/08584140406891133543\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2558848-svc/providers/Microsoft.Resources/deployments/cluster-d25sm5gw5h6au\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2558848-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2558848-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2558848-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2558848-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2558848-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-18815212-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2558848-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-18815212-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2558848-svc/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2558848-svc/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2558848-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2558848-svc/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2558848-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-21467642-vmss'. Please try alternative sizes 'Standard_D8ds_v6' (zones 2, 3), 'Standard_D4ads_v7' (zones 2, 1, 3), 'Standard_D4s_v6' (zones 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2558848-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-21467642-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091816375096971264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6429,"pr_state":"open","pr_sha":"d31f1425ab99ac4190688793ddf767c13e257968","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T09:14:22Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8fad62d4ddd68f37eecf06688408a239bb04500e5397d5b13a11c3953bc46b29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091816375096971264","occurred_at":"2026-08-24T09:14:22Z","failed_at":"provision","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-08-24T09:36:25.013Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T09:36:26.067Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T09:36:27.123Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=beaeaed83a7cf95a1b09858e4e28288a9e72f993cc66ca81ec00ff1e8e54f826 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6971264-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fbeaeaed83a7cf95a1b09858e4e28288a9e72f993cc66ca81ec00ff1e8e54f826\ntime=2026-08-24T09:44:28.907Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6971264-mgmt-2/providers/Microsoft.Resources/deployments/beaeaed83a7cf95a1b09858e4e28288a9e72f993cc66ca81ec00ff1e8e54f826/operationStatuses/08584140426988902505\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6971264-mgmt-2/providers/Microsoft.Resources/deployments/cluster-vig6u5v5k2tlc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6971264-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6971264-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6971264-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6971264-mgmt-2/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6971264-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-38714764-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6971264-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-38714764-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091814034880860160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6517,"pr_state":"open","pr_sha":"0d8584d01e41434248048570558d1041d0efaab9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T09:05:04Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ae6e993f2c71d7d2bbe938b2c7f7cd6d8dd79ebab97c671d7978f1a4b331eed5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091814034880860160","occurred_at":"2026-08-24T09:05:04Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T09:32:40.860Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T09:32:42.623Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T09:32:44.680Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=b5c54fde09a66b2790cd440c85cabb05b95df7e9c3a5cfe24051d2a16aa5cb1f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0860160-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb5c54fde09a66b2790cd440c85cabb05b95df7e9c3a5cfe24051d2a16aa5cb1f\ntime=2026-08-24T09:41:16.547Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0860160-svc/providers/Microsoft.Resources/deployments/b5c54fde09a66b2790cd440c85cabb05b95df7e9c3a5cfe24051d2a16aa5cb1f/operationStatuses/08584140429221841580\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0860160-svc/providers/Microsoft.Resources/deployments/cluster-5n4axvaafqqu4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0860160-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0860160-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0860160-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j0860160-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0860160-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-15189735-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0860160-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-15189735-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091812979992432640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6575,"pr_state":"open","pr_sha":"0633420cc119b5143a145776d79b9bd204c16df0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T09:00:53Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ef69ae36950aa030c94aae04849b576dab97af375c1a40981d8202d8a6904226","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091812979992432640","occurred_at":"2026-08-24T09:00:53Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T09:24:29.545Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T09:24:31.294Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T09:24:33.094Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=73468fd8f0c78c5427a4cf7823170085692bbf8052a8fc7aafe06063c9a01e4a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2432640-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F73468fd8f0c78c5427a4cf7823170085692bbf8052a8fc7aafe06063c9a01e4a\ntime=2026-08-24T09:32:34.962Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2432640-svc/providers/Microsoft.Resources/deployments/73468fd8f0c78c5427a4cf7823170085692bbf8052a8fc7aafe06063c9a01e4a/operationStatuses/08584140434133821616\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2432640-svc/providers/Microsoft.Resources/deployments/cluster-ii7fofvtmntgk\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2432640-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2432640-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2432640-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2432640-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2432640-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-26463169-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2432640-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-26463169-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2432640-svc/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2432640-svc/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2432640-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2432640-svc/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2432640-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-17010878-vmss'. Please try alternative sizes 'Standard_D8ds_v6' (zones 2, 3), 'Standard_D4ads_v7' (zones 2, 1, 3), 'Standard_D4s_v6' (zones 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2432640-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-17010878-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091811921836642304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6618,"pr_state":"open","pr_sha":"669f42e493665b25684e55d7e521b8c169ca1f38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T08:56:40Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b9ecf6e76dd0da64ef9ad140074cd803cbd1e2cf0350ba4998806a3d64da015a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091811921836642304","occurred_at":"2026-08-24T08:56:40Z","failed_at":"e2e","signature_id":"67e0d9c3e541426a700c2b1097cab5407e75837246f0b2492c0fa4c512883fb9","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005da4a0\u003e: \n DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005da4a0\u003e: \n DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6655/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091810349014585344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6655,"pr_state":"open","pr_sha":"318985320c7e33831c75975b37044b7f436fb67a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T08:50:25Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4e17cd1ae567029f8b9fde383b41a2e7c0e690eb4e6e321cdbe2906c7d8e38ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6655/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091810349014585344","occurred_at":"2026-08-24T08:50:25Z","failed_at":"provision","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-08-24T09:44:43.919Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T09:44:44.928Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-08-24T09:44:46.104Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=f40d7b7ab0b2d9646dc8b78fbaa252859442c0d4d4549c8765d66346f06ac123 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4585344-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff40d7b7ab0b2d9646dc8b78fbaa252859442c0d4d4549c8765d66346f06ac123\ntime=2026-08-24T10:14:43.424Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091803125458407424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6504,"pr_state":"open","pr_sha":"af0ab1f037eef8141b97aadd390cc54296b6d1c3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T08:21:34Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b4bf1d99bdff95b2265cc4a268e3c6c2a3c4112be7e03b593b4d58c3b783b89b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091803125458407424","occurred_at":"2026-08-24T08:21:34Z","failed_at":"provision","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-08-24T08:44:00.342Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T08:44:01.736Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T08:44:02.811Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=9d0846032d344f434503069fd459e7a4c951b5611b431c2eb0b0065a78e96029 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8407424-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9d0846032d344f434503069fd459e7a4c951b5611b431c2eb0b0065a78e96029\ntime=2026-08-24T08:52:34.725Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8407424-mgmt-2/providers/Microsoft.Resources/deployments/9d0846032d344f434503069fd459e7a4c951b5611b431c2eb0b0065a78e96029/operationStatuses/08584140458432325611\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8407424-mgmt-2/providers/Microsoft.Resources/deployments/cluster-xw6oxa26fzyyu\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8407424-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8407424-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8407424-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j8407424-mgmt-2/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8407424-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-92828001-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8407424-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-92828001-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091801419551084544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6504,"pr_state":"open","pr_sha":"af0ab1f037eef8141b97aadd390cc54296b6d1c3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T08:14:51Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9304026950531d0b68fd5d802b142a1d90c742027224f17e81fa657062865b19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091801419551084544","occurred_at":"2026-08-24T08:14:51Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-08-24T08:42:01.093Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T08:42:02.234Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-08-24T08:42:03.446Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=2e3eb598692cc106a7637db2eb5370285b0cbe96999e9aba188997911e08ff10 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1084544-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2e3eb598692cc106a7637db2eb5370285b0cbe96999e9aba188997911e08ff10\ntime=2026-08-24T08:50:05.190Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1084544-mgmt-1/providers/Microsoft.Resources/deployments/2e3eb598692cc106a7637db2eb5370285b0cbe96999e9aba188997911e08ff10/operationStatuses/08584140459626748434\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1084544-mgmt-1/providers/Microsoft.Resources/deployments/cluster-squivz75m522c\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1084544-mgmt-1/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1084544-mgmt-1/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1084544-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1084544-mgmt-1/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1084544-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-33704510-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1084544-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-33704510-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091795137024757760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6618,"pr_state":"open","pr_sha":"669f42e493665b25684e55d7e521b8c169ca1f38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T07:49:58Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fbebdf2e111421068da0d2d32912f6f1b974b7d48e81c593c513b253114d2a11","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091795137024757760","occurred_at":"2026-08-24T07:49:58Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T08:12:40.945Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T08:12:42.670Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T08:12:44.550Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=583e4878952541552a771b0cfcf02356ded24cbe2016bad8aa87f7a845c8c6c7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4757760-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F583e4878952541552a771b0cfcf02356ded24cbe2016bad8aa87f7a845c8c6c7\ntime=2026-08-24T08:21:46.727Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4757760-svc/providers/Microsoft.Resources/deployments/583e4878952541552a771b0cfcf02356ded24cbe2016bad8aa87f7a845c8c6c7/operationStatuses/08584140477221183467\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4757760-svc/providers/Microsoft.Resources/deployments/cluster-ijkmul6obanbq\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4757760-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4757760-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4757760-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j4757760-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4757760-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-32457879-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 2, 3), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4757760-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-32457879-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091793135989428224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6447,"pr_state":"open","pr_sha":"584ff7353b4a5ef8d593582c0a2a2c218bf10f82","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T07:42:01Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"172f4ed37069a60b782cf23c35dce336225ade4d2359be82ddafc432a6d601d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091793135989428224","occurred_at":"2026-08-24T07:42:01Z","failed_at":"provision","signature_id":"497ab79e6fe621da7f982b21f5dd28a45dc2f63297d3ed53abd84200879145f5","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nodepools (stamp=2)","test_suite":"step graph","failure_text":"time=2026-08-24T08:15:25.803Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nodepools stamp=2 description=\"Step nodepools\\n Kind: ARMStack\\n Template: templates/aks-nodepools.bicep\\n Parameters: configurations/mgmt-cluster-nodepools.tmpl.bicepparam\"\ntime=2026-08-24T08:18:16.150Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nodepools stamp=2 err=\"stamp 2: failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\\",\\n \\\"name\\\": \\\"67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\\",\\n \\\"status\\\": \\\"failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: '676c3fe4-b0a1-44f5-8602-c3c05012b379'.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2/providers/Microsoft.Resources/deployments/l4mv3scto7z6edigthkjdr62jbdrgoxoif33nvakxlwvg-260824084ipd0\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"OverconstrainedZonalAllocationRequest\\\",\\n \\\"message\\\": \\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-41975975-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-41975975-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"\\n }\\n ]\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36e7932077fe0de3697c04fb0ba302a7298b56f122f6c49db035b3d764e4980b","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentStackDeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36e7932077fe0de3697c04fb0ba302a7298b56f122f6c49db035b3d764e4980b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091792609700745216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6278,"pr_state":"closed","pr_sha":"ad8973f173b7ed9e4aeb0f1ff48f0cd895974ed1","final_merged_sha":"33f9055c00d2b52023c4da69e51807e8d141a7e5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T07:39:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5f969de68ffd83440e2f412553b9b21a241abf1b9ccf5b0ffa0117a8dfc36062","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091792609700745216","occurred_at":"2026-08-24T07:39:56Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T07:48:49.284Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T07:48:51.673Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T07:48:53.514Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=c4ffb8eb5fe59fa8f9c215105245e65567b3ae75c58f1894e0f72126f839af18 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0745216-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc4ffb8eb5fe59fa8f9c215105245e65567b3ae75c58f1894e0f72126f839af18\ntime=2026-08-24T07:57:25.627Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0745216-svc/providers/Microsoft.Resources/deployments/c4ffb8eb5fe59fa8f9c215105245e65567b3ae75c58f1894e0f72126f839af18/operationStatuses/08584140491530361585\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0745216-svc/providers/Microsoft.Resources/deployments/cluster-xzssy2rlha6dw\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0745216-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0745216-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0745216-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j0745216-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0745216-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-27534110-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0745216-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-27534110-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091783957136805888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6519,"pr_state":"open","pr_sha":"0e848b1b975748930d8aaaa9aa00622fa3dde7a3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-24T07:05:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091780518591860736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6278,"pr_state":"closed","pr_sha":"ad8973f173b7ed9e4aeb0f1ff48f0cd895974ed1","final_merged_sha":"33f9055c00d2b52023c4da69e51807e8d141a7e5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T06:51:53Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"99836dd421921b4e9f97319e74c284f9e980217118f5e2aa31d5b90c8d8aa4fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091780518591860736","occurred_at":"2026-08-24T06:51:53Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-08-24T07:13:06.981Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-08-24T07:13:09.018Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-08-24T07:13:10.536Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1860736%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59\ntime=2026-08-24T07:29:44.382Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1860736/providers/Microsoft.Resources/deployments/9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59/operationStatuses/08584140512958847265\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1860736/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1860736/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1860736\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Timeout; detail message Operation is being Canceled due to timeout.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6656/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091776267920609280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6656,"pr_state":"closed","pr_sha":"8d7f552861d56e8b6118f9edd6a7f5a6059ddf76","final_merged_sha":"8d7f552861d56e8b6118f9edd6a7f5a6059ddf76","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-24T06:35:00Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0e3aaa63e1858bac55a3c0bf2590a8781be4f41f8747039c098bda8fead0c4fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6656/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091776267920609280","occurred_at":"2026-08-24T06:35:00Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T06:57:54.785Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T06:57:57.820Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T06:57:59.587Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=183371dadaa86003cf4b2e1148d420a38ad735fb1aed96107d051f056b9bc826 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0609280-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F183371dadaa86003cf4b2e1148d420a38ad735fb1aed96107d051f056b9bc826\ntime=2026-08-24T07:06:31.480Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0609280-svc/providers/Microsoft.Resources/deployments/183371dadaa86003cf4b2e1148d420a38ad735fb1aed96107d051f056b9bc826/operationStatuses/08584140522069521911\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0609280-svc/providers/Microsoft.Resources/deployments/cluster-xfirgka634tem\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0609280-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0609280-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0609280-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j0609280-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0609280-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-45479501-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0609280-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-45479501-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0609280-svc/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0609280-svc/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0609280-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j0609280-svc/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0609280-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-40810365-vmss'. Please try alternative sizes 'Standard_D8ds_v6' (zones 3, 2), 'Standard_D4ads_v7' (zones 3, 1, 2), 'Standard_D4s_v6' (zones 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0609280-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-40810365-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091774986946613248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6643,"pr_state":"open","pr_sha":"086d6924efa1c3a9405352cb36cc160ddc7186fe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-24T06:29:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091774664165560320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T06:28:37Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"227a6bd5e2a69cdba3459af3d195a58507cf19011a674486c0d4db3c6029c8bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091774664165560320","occurred_at":"2026-08-24T06:28:37Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T07:03:34.745Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T07:03:36.353Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T07:03:37.884Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=5bdd2a18d3c4d97b3c3ef1c9b862a9a9b55771e7a1021879c9690a8a898ffb61 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5560320-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5bdd2a18d3c4d97b3c3ef1c9b862a9a9b55771e7a1021879c9690a8a898ffb61\ntime=2026-08-24T07:11:39.568Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5560320-svc/providers/Microsoft.Resources/deployments/5bdd2a18d3c4d97b3c3ef1c9b862a9a9b55771e7a1021879c9690a8a898ffb61/operationStatuses/08584140518685563513\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5560320-svc/providers/Microsoft.Resources/deployments/cluster-mssgpgtmbq63s\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5560320-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5560320-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5560320-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j5560320-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5560320-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-20641610-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5560320-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-20641610-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6655/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091767387383664640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6655,"pr_state":"open","pr_sha":"7b9035bce95ab309e4c40010eb6094f4a8be2ecb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-24T05:59:42Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091750724990930944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6503,"pr_state":"open","pr_sha":"6c5cfd09b9bbf1585b1cd37220da16677f327452","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T04:53:30Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"543cd62f2267063d9b4f06d899648374179029ea09665e8672aa906480670f0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091750724990930944","occurred_at":"2026-08-24T04:53:30Z","failed_at":"provision","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-08-24T05:19:23.165Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T05:19:24.420Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T05:19:25.641Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=fbb76f4ae15525e574d930a608886264a04bd969a6a0bd75915b35d608025287 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0930944-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ffbb76f4ae15525e574d930a608886264a04bd969a6a0bd75915b35d608025287\ntime=2026-08-24T05:26:57.916Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0930944-mgmt-2/providers/Microsoft.Resources/deployments/fbb76f4ae15525e574d930a608886264a04bd969a6a0bd75915b35d608025287/operationStatuses/08584140581204682703\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0930944-mgmt-2/providers/Microsoft.Resources/deployments/cluster-wdfog3tlntwm4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0930944-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0930944-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0930944-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0930944-mgmt-2/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0930944-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-39675041-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0930944-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-39675041-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091747201922371584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T04:39:30Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"20a12fbd29ed9cd37245b0296219788148ea8d2bde7fa61eada361fdb04dadd4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091747201922371584","occurred_at":"2026-08-24T04:39:30Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T04:48:17.406Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T04:48:19.724Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T04:48:21.678Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=2c1fd7cde1ae44aa6b38efdfdd841cda537f1304b159ec745060bb98e153aa01 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2371584-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2c1fd7cde1ae44aa6b38efdfdd841cda537f1304b159ec745060bb98e153aa01\ntime=2026-08-24T04:57:23.869Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2371584-svc/providers/Microsoft.Resources/deployments/2c1fd7cde1ae44aa6b38efdfdd841cda537f1304b159ec745060bb98e153aa01/operationStatuses/08584140599851363268\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2371584-svc/providers/Microsoft.Resources/deployments/cluster-6cvwmxyq3qdus\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2371584-svc/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2371584-svc/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2371584-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2371584-svc/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2371584-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-42381488-vmss'. Please try alternative sizes 'Standard_D8ds_v6' (zones 2, 3), 'Standard_D4ads_v7' (zones 2, 1, 3), 'Standard_D4s_v6' (zones 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2371584-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-42381488-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2371584-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2371584-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2371584-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2371584-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2371584-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-11454107-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2371584-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-11454107-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091745740643635200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6646,"pr_state":"closed","pr_sha":"c0a69cc848be7d730ae55bfa6100b03fe7f76bce","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-08-24T04:33:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091734160543584256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6583,"pr_state":"open","pr_sha":"515e34b8a6585d1a0a514e51a994786b923c61be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T03:47:41Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2be637d4c950e1748f9b049160b15acafaf57f338d61fd2552422c673eac4a54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091734160543584256","occurred_at":"2026-08-24T03:47:41Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-08-24T04:17:24.903Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T04:17:27.266Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T04:17:29.064Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=6967dee589ade651c69bb96507e6cb0c7cb21cecf78950dae52a9a9a4b095938 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3584256-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6967dee589ade651c69bb96507e6cb0c7cb21cecf78950dae52a9a9a4b095938\ntime=2026-08-24T04:26:00.988Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3584256-svc/providers/Microsoft.Resources/deployments/6967dee589ade651c69bb96507e6cb0c7cb21cecf78950dae52a9a9a4b095938/operationStatuses/08584140618375600165\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3584256-svc/providers/Microsoft.Resources/deployments/cluster-skudbwx3rh3io\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3584256-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3584256-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3584256-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j3584256-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3584256-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-14718963-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3584256-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-14718963-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091733541242015744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6646,"pr_state":"closed","pr_sha":"c0a69cc848be7d730ae55bfa6100b03fe7f76bce","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-08-24T03:45:13Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2dc1bf8c40a0be55c7ed94737e03c4383842d69b3422fc252b8c63d722c9acba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091733541242015744","occurred_at":"2026-08-24T03:45:13Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-08-24T04:16:06.917Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T04:16:08.068Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-08-24T04:16:10.699Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=b90b7bfd10ed0c59be2c1babf2e237ad81ed58d2c1a0b3b120a0a0e4de3596a0 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2015744-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb90b7bfd10ed0c59be2c1babf2e237ad81ed58d2c1a0b3b120a0a0e4de3596a0\ntime=2026-08-24T04:23:42.372Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2015744-mgmt-1/providers/Microsoft.Resources/deployments/b90b7bfd10ed0c59be2c1babf2e237ad81ed58d2c1a0b3b120a0a0e4de3596a0/operationStatuses/08584140619164865579\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2015744-mgmt-1/providers/Microsoft.Resources/deployments/cluster-mm4ibqsrqe2za\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2015744-mgmt-1/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2015744-mgmt-1/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2015744-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j2015744-mgmt-1/agentPools/infra1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2015744-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-28289844-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2015744-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-28289844-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"int","summary":{"total_runs":6,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":4,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2092019235080376320","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-24T22:40:28Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091961854199861248","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-24T18:52:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091914179135934464","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-24T15:42:43Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091834116805758976","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-24T10:24:52Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"26b7d88074ebfdd37efe1b4ee6f80e0f7415e6a240754054eb16c60eec8e9680","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091834116805758976","occurred_at":"2026-08-24T10:24:52Z","failed_at":"e2e","signature_id":"c4114fce72966a1339b4e9230a4f088e9e9d8e5a8939f917174c276b7d0a2542","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handshake timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handshake timeout","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d414eaa781a1a73b840b0ceb0c494d2cfb36d6ce2f01987a892eec84026f3876","failure_pattern":"failed to select node pool \u003cnodepool\u003e version for \u003cocp-channel\u003e channel: Get \"\u003curl\u003e\": net/http: TLS handsh","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handsh"}},{"row_id":"cea05f9ecb0eb3bf62f9d27a165ecdbae4935b9b40688b9bfcbc2c02019b341d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091834116805758976","occurred_at":"2026-08-24T10:24:52Z","failed_at":"e2e","signature_id":"3fa15812aed9d226c2b6bdead73b3a2c80d16497f78ee163ab8cbfa1074904ed","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e43900\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fbxfthcgbs8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n \"name\": \"7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T10:43:39.73808157Z\",\n \"endTime\": \"2026-08-24T11:02:45.476245961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e43900\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fbxfthcgbs8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n \"name\": \"7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T10:43:39.73808157Z\",\n \"endTime\": \"2026-08-24T11:02:45.476245961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13ca13618a2c4b8b7429b3001f7d02e419916c0961928bfe37af66198711655d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["13ca13618a2c4b8b7429b3001f7d02e419916c0961928bfe37af66198711655d","d414eaa781a1a73b840b0ceb0c494d2cfb36d6ce2f01987a892eec84026f3876"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091798589079556096","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-24T08:03:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091735435544891392","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-24T03:52:45Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"141bd8e8e1b1dbff08fca711fa7f20149c11e7e569e3316a7b69f9c2845d8103","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091735435544891392","occurred_at":"2026-08-24T03:52:45Z","failed_at":"e2e","signature_id":"571e7c77690730a14fa42704d42877387a3fe5f21673f5f238b11729008a3c86","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-l8xph5\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f20140\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-trf9f.apps.aro.r8g4l2d8f3g0v2i.b5du.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-l8xph5\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f20140\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-trf9f.apps.aro.r8g4l2d8f3g0v2i.b5du.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-24T23:00:55Z"},"failed_at":["e2e"],"failed_test_count":7,"occurrences":[{"row_id":"02c9c7a94092934af361527dc6d311afb1832fe762e738a0d5a155bd980aac30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"498a13bc157b497dedc0c407f7c032fb09b28d4110c6fbee95fdae33ff5fe349","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-cbj2dm\" to 4.22.0-0.nightly-multi-2026-08-23-080550\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df2720\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-cbj2dm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4ls4ksxqchgg\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n \"name\": \"d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:48:14.416516082Z\",\n \"endTime\": \"2026-08-24T23:48:20.787265431Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-cbj2dm\" to 4.22.0-0.nightly-multi-2026-08-23-080550\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df2720\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-cbj2dm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4ls4ksxqchgg\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n \"name\": \"d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:48:14.416516082Z\",\n \"endTime\": \"2026-08-24T23:48:20.787265431Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d8c04d75e2d056f1b8b36668a13ab7045b056c9f740c7013c4882cfcfef5d2d6","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"085bab2409f9f5947bcab97e70cb7f7668ac23ec25719c1cf553f242f84a78f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"15bac440b3d4b1dd68415a557e13f0d806f6cdb816f9edcd9423e02924b01f2e","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-7z565t\" to 4.21.0-0.nightly-multi-2026-08-24-143120\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b5de0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-7z565t\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-98477z6czjf7\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n \"name\": \"d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:45:23.595556181Z\",\n \"endTime\": \"2026-08-24T23:45:30.682388649Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-7z565t\" to 4.21.0-0.nightly-multi-2026-08-24-143120\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b5de0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-7z565t\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-98477z6czjf7\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n \"name\": \"d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:45:23.595556181Z\",\n \"endTime\": \"2026-08-24T23:45:30.682388649Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f966d8a6f1dc892e4c6ee21571e53ba7d8d1ed39b375430e469f3616f54b9b95","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"19ce708d2233c4841b683f4f3cb377dbcf2d099d6f244583b40ea9838b4bdc94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"48cf5ec4badd8d65b424a299cc4f30c373af95f5d42c09e6725e0ee15b8c1667","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001f06450\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-m4qs88j7zlw8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001f06450\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-m4qs88j7zlw8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-gm6hwh, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: failed waiting for cluster=\"cluster-candidate-5-0-gm6hwh\" in resourcegroup=\"rg-candidate-5-0-mqrq7j46wgxv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: context deadline exceeded"}},{"row_id":"3f7ce80ba2078b9411717d5f3982325b2a505921eb43e547f0541191896849b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"6663bc04df6e96cb243dd9f35fd825b118edab670cd360f1ba5fc79ade7a0f44","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-lrdz8v\" to 4.23.0-0.nightly-multi-2026-08-24-110923\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000881340\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-lrdz8v\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-t25q8kvqvdhw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\",\n \"name\": \"4393d587-2d85-4674-a4f6-155ceb1138f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:48:28.850341951Z\",\n \"endTime\": \"2026-08-24T23:48:30.944709051Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-lrdz8v\" to 4.23.0-0.nightly-multi-2026-08-24-110923\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000881340\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-lrdz8v\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-t25q8kvqvdhw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\",\n \"name\": \"4393d587-2d85-4674-a4f6-155ceb1138f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:48:28.850341951Z\",\n \"endTime\": \"2026-08-24T23:48:30.944709051Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"724e7a51f2cd05d0f0c351e24f28344ba4eb40490e6de7ed02ace0e7a7856472","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"44bbaf32998d440593c42841fbdfedb10b26afb76a63b3b2704c6804305ed7f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"05526aa86d5e4dd680e08fb3cf223532281a30bb281575b5c8abfec129525d60","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-q6hwb9\" to 5.1.0-0.nightly-multi-2026-08-24-111752\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a25da0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-q6hwb9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-kgxwzfcjcc2b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n \"name\": \"7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:51:24.1449981Z\",\n \"endTime\": \"2026-08-24T23:51:30.86207539Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-q6hwb9\" to 5.1.0-0.nightly-multi-2026-08-24-111752\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a25da0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-q6hwb9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-kgxwzfcjcc2b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n \"name\": \"7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:51:24.1449981Z\",\n \"endTime\": \"2026-08-24T23:51:30.86207539Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3b509588f678a0fd976eb453004a3e8a55c0be5c12f8340c463453a657ce6678","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b6ddbd889400218760770b2c79af341a3a6a19c084736862e30c7a5a55282f3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"0c1e40a7688ce333e1802f99689a7431268dce71db699919518e6ab02caaba1e","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002b4c960\u003e: \n node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1.35.4\"); o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002b4c960\u003e: \n node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1.35.4\"); o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6411b2b281a97b7e65400b269143638faa6befdec2dd319ac1327622333be049","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1"}},{"row_id":"e50f9d8548281ee93b0655e818beebfadadfea40801c76a50587dd967b3c9fbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"a75e2b44cb494e9ca286b15ddee1a744c32c76fd89cb2c425050bd89a5961c7e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-c58j4775b54x/cluster-nightly-5-1-rb2jwr should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001a0d800\u003e: \n failed to create HCP cluster cluster-nightly-5-1-rb2jwr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: failed waiting for cluster=\"cluster-nightly-5-1-rb2jwr\" in resourcegroup=\"rg-nightly-5-1-c58j4775b54x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-c58j4775b54x/cluster-nightly-5-1-rb2jwr should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001a0d800\u003e: \n failed to create HCP cluster cluster-nightly-5-1-rb2jwr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: failed waiting for cluster=\"cluster-nightly-5-1-rb2jwr\" in resourcegroup=\"rg-nightly-5-1-c58j4775b54x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-gm6hwh, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: failed waiting for cluster=\"cluster-candidate-5-0-gm6hwh\" in resourcegroup=\"rg-candidate-5-0-mqrq7j46wgxv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","3b509588f678a0fd976eb453004a3e8a55c0be5c12f8340c463453a657ce6678","6411b2b281a97b7e65400b269143638faa6befdec2dd319ac1327622333be049","724e7a51f2cd05d0f0c351e24f28344ba4eb40490e6de7ed02ace0e7a7856472","d8c04d75e2d056f1b8b36668a13ab7045b056c9f740c7013c4882cfcfef5d2d6","f966d8a6f1dc892e4c6ee21571e53ba7d8d1ed39b375430e469f3616f54b9b95"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-24T02:00:54Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"66b24c3f50b0870d8aa6936fe785d9a6de4a1e012265bf94ff613fc9f6277b25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"3fd31e6b902a75b339639f3cb28f3e152b295a16939ec2f7056ea2da7a8f4e73","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000ad6720\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000ad6720\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6c0df54cac07833305ae54d7a30bcf9a55390b618040466bd5f2843d80e698a8","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5"}},{"row_id":"8b3ebf9976c07824dca0eedd2a57a3a7a60bb8c04b2e411b74353848ab355bfd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"eabd42ab0d34b7f1d8ca082eb227c6c5a44990af4cb2ad49ebe668cc8663e6ed","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-znvqfb\" to minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0008e8630\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-bb24w6nmgn9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-znvqfb\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-znvqfb\" to minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0008e8630\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-bb24w6nmgn9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-znvqfb\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5443585aedc88f5d2589a8fb3795e77266206bcfdb680d3cfbedccaf7bc67f50","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.30; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"9ffa4b8b219b2f110bed78bd8cdfb1881e0f2d74529adbbd0cef680c2f9732f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"233817672ec9f15c61036dd69be2232e5da454215c4ae03262d5d9d1147fe964","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-gmlx25\" to minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000703a40\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b6dm7hck8ksd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-gmlx25\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-gmlx25\" to minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000703a40\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b6dm7hck8ksd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-gmlx25\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2b6de8e3eeb7b1eb9cb9733a09ec9f907952649e2f47b910e6bb29761a829864","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.10; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"a3f851449d9987ed815c4a41efc3f200e2253b45a9009c8a61a468ea87165dcc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"67be8c6caed95a193a1745532d025891093937507a496524f899a7d2c80f89f2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-96fcvx\" to minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0008e5080\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-qkzx8x2fwwlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-96fcvx\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-96fcvx\" to minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0008e5080\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-qkzx8x2fwwlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-96fcvx\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"beaf47bd37dc21f47808a0240288ea4322da110d92ca67bf08de0f6793353753","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.35; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"f3da9628dd8fe151e0a430874769a4e6c7d5028caf9c410d91690d351ca958ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"5fc7819289b1249d9f2299f61167ea9b6a16c06d2102e06da9f3562ee86b0ad0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-mqrq7j46wgxv/cluster-candidate-5-0-gm6hwh should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001400690\u003e: \n failed to create HCP cluster cluster-candidate-5-0-gm6hwh, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: failed waiting for cluster=\"cluster-candidate-5-0-gm6hwh\" in resourcegroup=\"rg-candidate-5-0-mqrq7j46wgxv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-mqrq7j46wgxv/cluster-candidate-5-0-gm6hwh should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001400690\u003e: \n failed to create HCP cluster cluster-candidate-5-0-gm6hwh, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: failed waiting for cluster=\"cluster-candidate-5-0-gm6hwh\" in resourcegroup=\"rg-candidate-5-0-mqrq7j46wgxv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-gm6hwh, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: failed waiting for cluster=\"cluster-candidate-5-0-gm6hwh\" in resourcegroup=\"rg-candidate-5-0-mqrq7j46wgxv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","2b6de8e3eeb7b1eb9cb9733a09ec9f907952649e2f47b910e6bb29761a829864","5443585aedc88f5d2589a8fb3795e77266206bcfdb680d3cfbedccaf7bc67f50","6c0df54cac07833305ae54d7a30bcf9a55390b618040466bd5f2843d80e698a8","beaf47bd37dc21f47808a0240288ea4322da110d92ca67bf08de0f6793353753"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"stg","summary":{"total_runs":3,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-24T23:00:55Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"008600a18ccd827a3042b4cd5fb4a648475322f1885505a8be83846816542afd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"6a0460b71f8ba99286143da10d9b371325b39609e9612fa043b5980a861407e3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc00118e480\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-h759ch\" in resourcegroup=\"rg-nightly-4-20-mzr4r5lz42qp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\",\n \"name\": \"55ed743f-d106-4adb-b590-caa3369caeb3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:25:48.884097042Z\",\n \"endTime\": \"2026-08-24T23:49:50.709200875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd7vm7aq4os1ksjghr3o1msj3ksno6l still exists (deletion dispatched at 2026-08-24T23:25:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc00118e480\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-h759ch\" in resourcegroup=\"rg-nightly-4-20-mzr4r5lz42qp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\",\n \"name\": \"55ed743f-d106-4adb-b590-caa3369caeb3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:25:48.884097042Z\",\n \"endTime\": \"2026-08-24T23:49:50.709200875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd7vm7aq4os1ksjghr3o1msj3ksno6l still exists (deletion dispatched at 2026-08-24T23:25:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1265afe1fd2b13d19844254b36072a8aacc52289d043db93b5e26c1f34415e07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"1ecfdcaa8fd2583c7f65498931069ae0e42fa1b933a7ed7714a974ea82a7b373","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002ac4480\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002ac4480\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded"}},{"row_id":"34717acd9b65a750fca11438f4deb64af0f1368425a560ba6ee080177d7a0588","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"ce363ec1f1315846ac4ca18a663f8ca57451d8f5854815877a996fbdf3f45dd9","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8ea40\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf6jg45wtwxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\",\n \"name\": \"62db7451-86ec-4c6a-9227-bc775f891ebb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:05:01.131009318Z\",\n \"endTime\": \"2026-08-24T23:24:05.637766062Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8ea40\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf6jg45wtwxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\",\n \"name\": \"62db7451-86ec-4c6a-9227-bc775f891ebb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:05:01.131009318Z\",\n \"endTime\": \"2026-08-24T23:24:05.637766062Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"73c8344b8caa60176d9a58a89cacbd56f5859697ebbc11abf17538f5eca3aa7b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"8c228d79217e5703c43c091d1bbcb6c74c0d2e48dd70a8ed3bbc113b8600d236","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001423c38\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-24-143120, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001423c38\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-24-143120, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"7e7589a04210dd654eed9c4626d2f4e66beeeb038504ceed3961527252064d29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"1f961d63e5efbc9f674297ab6ff360e18191616cad68297a32bfc0352b1efd45","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0012c61b0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-23-080550, targetMinor=4.23.0-0.nightly-multi-2026-08-24-110923) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-24-110923\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0012c61b0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-23-080550, targetMinor=4.23.0-0.nightly-multi-2026-08-24-110923) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-24-110923\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"7e89588dd3937379d7d3bf76223646a84a8d1bfa6f3e5b041f578281aca534c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"5565031e089c6eb68fe43eb98d2d43776a82305498e8c6a0a7f4c918b0348bf0","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0012c0d10\u003e: \n node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]},{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0012c0d10\u003e: \n node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]},{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c75b40567502299dd397c0248cce6097e2377df763d3deec7ae8c7f9cd09e3b2","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1"}},{"row_id":"8f880d5ec359cc3461d3f3eca8626a5f31e0b0bfcae65d728773a2efeda75d86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"5d51d6400541f1ef2ea735bc34f88c2cca7f1b91d8f8680efed7dfe0ab6e075d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000013c68\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-24-135049, targetMinor=4.21.0-0.nightly-multi-2026-08-24-143120) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-24-143120\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000013c68\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-24-135049, targetMinor=4.21.0-0.nightly-multi-2026-08-24-143120) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-24-143120\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"939f447d8628307ab6183a419b4c90bec3d9e6573e924d08731680757f32f597","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"9a6af90baac0bf0367bb530e542307559248b48af09dc4fa5550242ec09f64e4","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000012588\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-24-152026, targetMinor=5.1.0-0.nightly-multi-2026-08-24-111752) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-24-111752\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000012588\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-24-152026, targetMinor=5.1.0-0.nightly-multi-2026-08-24-111752) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-24-111752\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"b403c77a765eedb7fa90d1d15cc0d5f0fa13adbdd2889bc4d6eee25e44bddba4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","failed_at":"e2e","signature_id":"3feed0e49b703d23a4823f602dec7fe62c46bf78411b06f90b5ee2c2bde72da5","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001716270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nml9dcvzzfr9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n \"name\": \"d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:14:11.789973414Z\",\n \"endTime\": \"2026-08-24T23:38:15.678890881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001716270\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nml9dcvzzfr9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n \"name\": \"d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-24T23:14:11.789973414Z\",\n \"endTime\": \"2026-08-24T23:38:15.678890881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"db86d0a7b49ef93daf7cdb63d12e83b79ddcd71210e2847a436b8d80dd5dde7f","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] \u003cno_message\u003e; [clusterServiceStatus] \u003cno_message\u003e; [descendantResources] \u003cno_message\u003e; [hostedCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","c75b40567502299dd397c0248cce6097e2377df763d3deec7ae8c7f9cd09e3b2","db86d0a7b49ef93daf7cdb63d12e83b79ddcd71210e2847a436b8d80dd5dde7f"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2091964346321080320","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-24T19:02:21Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-24T02:00:54Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"0fffce43c73a55dcb3d4bd28f5e502c696cadabff3626c665c1d35f1942b8bb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"6f400eea163c9e3d28c8b1f64accc1cf5963abf78c260413f5fe388a60f0006c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-xcnwcj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012d43e0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-4fwk5.apps.aro.j2u3a0g3c2u7a6b.ug6p.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-xcnwcj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012d43e0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-4fwk5.apps.aro.j2u3a0g3c2u7a6b.ug6p.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"1456be3b126df8094458ade3844ed597217601bf2e4e8c2566c960572ca2b571","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"c14f9a5639d319819baa7a308cbb789519348e32b0abac6b02f5611b9187fbe7","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0017be510\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0017be510\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b6258cd488b080bf5ba7099ce8387b9d2f4d222d926f9a0c81022dc736e89db2","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil"}},{"row_id":"1602977d98b0be3a7f48e0b8855e61aff4342243ef02c00fa5c6064de418af49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"64d7370897c1ddb0a4c082a98b5de0abd894ee5bd836383e651a1f53f91a033d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-699qgk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f80a40\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-5gnbj.apps.aro.v3f9c8v2c1l5f3b.qmeg.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-699qgk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f80a40\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-5gnbj.apps.aro.v3f9c8v2c1l5f3b.qmeg.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"249169d4ec94beda16011254572212424e7663b6be18c27fe633648094917ef5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003cx509.UnknownAuthorityError\u003e: \n x509: certificate signed by unknown authority\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003cx509.UnknownAuthorityError\u003e: \n x509: certificate signed by unknown authority\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority"}},{"row_id":"3ca9c8d18e00c402b6249a895c59da72e1319d826b0ca5b6c26843bdddad946c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"de74f8de034a6e8da10a240f9e0d6f03ee3be2522666aaf0b6ad709ddf3efbc4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-7w6n6t\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e32420\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-5pxzh.apps.aro.k9t7j1s6j4b3x8l.omcf.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-7w6n6t\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e32420\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-5pxzh.apps.aro.k9t7j1s6j4b3x8l.omcf.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"8114d7dc044296833a8e50d2de3b9b65414c8fd11480bb195591e6f803818fdf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"568397239d373e84cc1193077022f2929fdbc7536d41e0a023cda085a91923ff","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc002bc60a8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2xkbq.apps.aro.cilium-cl.yqg3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc002bc60a8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2xkbq.apps.aro.cilium-cl.yqg3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"8f310c16aa395d3c9fe92d11beff706ff6c94bc90137af99315fc098f995f2b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"268af4cfefc9d9ce5477c6309e34d9e625934c5fec5612737f144e143459ca23","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc0023089a8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gd2m6.apps.aro.agg-cluster.sh9x.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc0023089a8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gd2m6.apps.aro.agg-cluster.sh9x.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"aee197628bb4da8a1608181e68db36d6c91166386b9d14172e6e901e76e723b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"aa4dc686c4f2ffb192f550321a5d618c671a84350ba22f26b4ee1aaac5686fb0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-mkwm4k\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001088c60\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-mcnh9.apps.aro.l9y2e4v2l1k5w6z.05b8.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-mkwm4k\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001088c60\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-mcnh9.apps.aro.l9y2e4v2l1k5w6z.05b8.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"c606f2a73e85917afe2499bc341bd3014a94a2498d99040d2ea4d5d0f26b802c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"c217516f086c4ee518f66b5fc0b63d7e19723573004fb051ee9ac9e2e83e0fb9","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc0010c1608\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-q8jjw.apps.aro.x9h1g1p0v0c2r9d.841h.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc0010c1608\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-q8jjw.apps.aro.x9h1g1p0v0c2r9d.841h.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"d8b6233a460834e5398fc4fff87c06a55b15de2e0cd7a732d12bc8d3b2253193","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","failed_at":"e2e","signature_id":"128bdc5bd9fd4d676037c7b3876358423100506abaac1cb7c6a2aa6dca99f9d8","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001990018\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-58vq7.apps.aro.cilium-cluster.xjbu.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001990018\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-58vq7.apps.aro.cilium-cluster.xjbu.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","b6258cd488b080bf5ba7099ce8387b9d2f4d222d926f9a0c81022dc736e89db2"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}