{"meta":{"start_date":"2026-08-24","end_date":"2026-08-24","start_at":"2026-08-24T00:00:00Z","end_at":"2026-08-25T00:00:00Z","timezone":"UTC","generated_at":"2026-08-25T22:51:00Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":103,"failed_runs":90,"raw_occurrences":190,"matched_occurrences":183,"runs_affected":83},"rows":[{"environment":"dev","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":35,"runs_affected":35,"failed_runs":35,"run_impact_percent":33.980582524271846,"anchor_occurrences":35,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,35],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":35,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":35}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b66310\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[00:48:59.861]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:48:54.9335446Z);\\nlet _endTime = datetime(2026-08-25T00:48:54.9335452Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:48:59.8208131Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45\\\",\\r\\n            \\\"activityId\\\": \\\"3f29cef9-55f3-4e39-a44c-dc2745aeb329\\\",\\r\\n            \\\"subActivityId\\\": \\\"9dc2e9b4-43ec-49f2-b454-4f12f219c5db\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"9b9edaeb-e277-470d-a49a-ba5e0f233409\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45 ARID=3f29cef9-55f3-4e39-a44c-dc2745aeb329 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9b9edaeb-e277-470d-a49a-ba5e0f233409 \\u003e DN.FE.ExecuteQuery/9dc2e9b4-43ec-49f2-b454-4f12f219c5db)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:48:59.984]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:48:54.9335446Z);\\nlet _endTime = datetime(2026-08-25T00:48:54.9335452Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:48:59.8208131Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45\\\",\\r\\n            \\\"activityId\\\": \\\"3f29cef9-55f3-4e39-a44c-dc2745aeb329\\\",\\r\\n            \\\"subActivityId\\\": \\\"9dc2e9b4-43ec-49f2-b454-4f12f219c5db\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"9b9edaeb-e277-470d-a49a-ba5e0f233409\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45 ARID=3f29cef9-55f3-4e39-a44c-dc2745aeb329 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9b9edaeb-e277-470d-a49a-ba5e0f233409 \\u003e DN.FE.ExecuteQuery/9dc2e9b4-43ec-49f2-b454-4f12f219c5db)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[00:49:04.412]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:00.0668173Z);\\nlet _endTime = datetime(2026-08-25T00:49:00.0668177Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:04.3919639Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef\\\",\\r\\n            \\\"activityId\\\": \\\"590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea\\\",\\r\\n            \\\"subActivityId\\\": \\\"f16f10a7-70e1-48c7-9e9d-209207cfc1e8\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"69739f9c-88ef-46ee-93e1-65af7b0a2416\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef ARID=590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/69739f9c-88ef-46ee-93e1-65af7b0a2416 \\u003e DN.FE.ExecuteQuery/f16f10a7-70e1-48c7-9e9d-209207cfc1e8)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:49:04.464]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:00.0668173Z);\\nlet _endTime = datetime(2026-08-25T00:49:00.0668177Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:04.3919639Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef\\\",\\r\\n            \\\"activityId\\\": \\\"590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea\\\",\\r\\n            \\\"subActivityId\\\": \\\"f16f10a7-70e1-48c7-9e9d-209207cfc1e8\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"69739f9c-88ef-46ee-93e1-65af7b0a2416\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef ARID=590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/69739f9c-88ef-46ee-93e1-65af7b0a2416 \\u003e DN.FE.ExecuteQuery/f16f10a7-70e1-48c7-9e9d-209207cfc1e8)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[00:49:12.903]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:08.6625258Z);\\nlet _endTime = datetime(2026-08-25T00:49:08.6625262Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:12.8815919Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;a5c94555-e249-4511-a807-b8477933031f\\\",\\r\\n            \\\"activityId\\\": \\\"3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c\\\",\\r\\n            \\\"subActivityId\\\": \\\"af4bd91d-24d7-4768-a750-045fb74a6811\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a5c94555-e249-4511-a807-b8477933031f ARID=3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa \\u003e DN.FE.ExecuteQuery/af4bd91d-24d7-4768-a750-045fb74a6811)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:49:13.008]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:08.6625258Z);\\nlet _endTime = datetime(2026-08-25T00:49:08.6625262Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:12.8815919Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;a5c94555-e249-4511-a807-b8477933031f\\\",\\r\\n            \\\"activityId\\\": \\\"3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c\\\",\\r\\n            \\\"subActivityId\\\": \\\"af4bd91d-24d7-4768-a750-045fb74a6811\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a5c94555-e249-4511-a807-b8477933031f ARID=3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa \\u003e DN.FE.ExecuteQuery/af4bd91d-24d7-4768-a750-045fb74a6811)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b66310\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[00:48:59.861]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:48:54.9335446Z);\\nlet _endTime = datetime(2026-08-25T00:48:54.9335452Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:48:59.8208131Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45\\\",\\r\\n            \\\"activityId\\\": \\\"3f29cef9-55f3-4e39-a44c-dc2745aeb329\\\",\\r\\n            \\\"subActivityId\\\": \\\"9dc2e9b4-43ec-49f2-b454-4f12f219c5db\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"9b9edaeb-e277-470d-a49a-ba5e0f233409\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45 ARID=3f29cef9-55f3-4e39-a44c-dc2745aeb329 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9b9edaeb-e277-470d-a49a-ba5e0f233409 \\u003e DN.FE.ExecuteQuery/9dc2e9b4-43ec-49f2-b454-4f12f219c5db)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:48:59.984]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:48:54.9335446Z);\\nlet _endTime = datetime(2026-08-25T00:48:54.9335452Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:48:59.8208131Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45\\\",\\r\\n            \\\"activityId\\\": \\\"3f29cef9-55f3-4e39-a44c-dc2745aeb329\\\",\\r\\n            \\\"subActivityId\\\": \\\"9dc2e9b4-43ec-49f2-b454-4f12f219c5db\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"9b9edaeb-e277-470d-a49a-ba5e0f233409\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45 ARID=3f29cef9-55f3-4e39-a44c-dc2745aeb329 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9b9edaeb-e277-470d-a49a-ba5e0f233409 \\u003e DN.FE.ExecuteQuery/9dc2e9b4-43ec-49f2-b454-4f12f219c5db)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[00:49:04.412]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:00.0668173Z);\\nlet _endTime = datetime(2026-08-25T00:49:00.0668177Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:04.3919639Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef\\\",\\r\\n            \\\"activityId\\\": \\\"590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea\\\",\\r\\n            \\\"subActivityId\\\": \\\"f16f10a7-70e1-48c7-9e9d-209207cfc1e8\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"69739f9c-88ef-46ee-93e1-65af7b0a2416\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef ARID=590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/69739f9c-88ef-46ee-93e1-65af7b0a2416 \\u003e DN.FE.ExecuteQuery/f16f10a7-70e1-48c7-9e9d-209207cfc1e8)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:49:04.464]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:00.0668173Z);\\nlet _endTime = datetime(2026-08-25T00:49:00.0668177Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:04.3919639Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef\\\",\\r\\n            \\\"activityId\\\": \\\"590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea\\\",\\r\\n            \\\"subActivityId\\\": \\\"f16f10a7-70e1-48c7-9e9d-209207cfc1e8\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"69739f9c-88ef-46ee-93e1-65af7b0a2416\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef ARID=590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/69739f9c-88ef-46ee-93e1-65af7b0a2416 \\u003e DN.FE.ExecuteQuery/f16f10a7-70e1-48c7-9e9d-209207cfc1e8)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[00:49:12.903]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:08.6625258Z);\\nlet _endTime = datetime(2026-08-25T00:49:08.6625262Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:12.8815919Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;a5c94555-e249-4511-a807-b8477933031f\\\",\\r\\n            \\\"activityId\\\": \\\"3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c\\\",\\r\\n            \\\"subActivityId\\\": \\\"af4bd91d-24d7-4768-a750-045fb74a6811\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a5c94555-e249-4511-a807-b8477933031f ARID=3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa \\u003e DN.FE.ExecuteQuery/af4bd91d-24d7-4768-a750-045fb74a6811)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:49:13.008]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:08.6625258Z);\\nlet _endTime = datetime(2026-08-25T00:49:08.6625262Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:12.8815919Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;a5c94555-e249-4511-a807-b8477933031f\\\",\\r\\n            \\\"activityId\\\": \\\"3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c\\\",\\r\\n            \\\"subActivityId\\\": \\\"af4bd91d-24d7-4768-a750-045fb74a6811\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a5c94555-e249-4511-a807-b8477933031f ARID=3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa \\u003e DN.FE.ExecuteQuery/af4bd91d-24d7-4768-a750-045fb74a6811)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00083c410\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[00:23:58.357]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:55.0351655Z);\\nlet _endTime = datetime(2026-08-25T00:23:55.0351661Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:23:58.3330999Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b\\\",\\r\\n            \\\"activityId\\\": \\\"442abb20-5490-45f5-ba70-3bca3bed0639\\\",\\r\\n            \\\"subActivityId\\\": \\\"b9132e8a-1c55-4f09-8982-f344852f0eee\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"99a04993-dace-423f-a879-2fbeee2a7b84\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b ARID=442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.Http.CallContext/442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.ExecuteQuery/99a04993-dace-423f-a879-2fbeee2a7b84 \\u003e DN.FE.ExecuteQuery/b9132e8a-1c55-4f09-8982-f344852f0eee)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:23:58.411]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:55.0351655Z);\\nlet _endTime = datetime(2026-08-25T00:23:55.0351661Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:23:58.3330999Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b\\\",\\r\\n            \\\"activityId\\\": \\\"442abb20-5490-45f5-ba70-3bca3bed0639\\\",\\r\\n            \\\"subActivityId\\\": \\\"b9132e8a-1c55-4f09-8982-f344852f0eee\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"99a04993-dace-423f-a879-2fbeee2a7b84\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b ARID=442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.Http.CallContext/442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.ExecuteQuery/99a04993-dace-423f-a879-2fbeee2a7b84 \\u003e DN.FE.ExecuteQuery/b9132e8a-1c55-4f09-8982-f344852f0eee)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[00:24:01.787]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:58.4821435Z);\\nlet _endTime = datetime(2026-08-25T00:23:58.4821442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:01.7618170Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5\\\",\\r\\n            \\\"activityId\\\": \\\"d2adae1e-7609-4016-b166-5a8ff8be55d8\\\",\\r\\n            \\\"subActivityId\\\": \\\"7c0e685c-e2d3-436f-89bc-9d9fa3141b61\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"8cf560a3-f257-4d9d-900b-aaf46185fe8b\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5 ARID=d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.Http.CallContext/d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.ExecuteQuery/8cf560a3-f257-4d9d-900b-aaf46185fe8b \\u003e DN.FE.ExecuteQuery/7c0e685c-e2d3-436f-89bc-9d9fa3141b61)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:24:01.845]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:58.4821435Z);\\nlet _endTime = datetime(2026-08-25T00:23:58.4821442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:01.7618170Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5\\\",\\r\\n            \\\"activityId\\\": \\\"d2adae1e-7609-4016-b166-5a8ff8be55d8\\\",\\r\\n            \\\"subActivityId\\\": \\\"7c0e685c-e2d3-436f-89bc-9d9fa3141b61\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"8cf560a3-f257-4d9d-900b-aaf46185fe8b\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5 ARID=d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.Http.CallContext/d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.ExecuteQuery/8cf560a3-f257-4d9d-900b-aaf46185fe8b \\u003e DN.FE.ExecuteQuery/7c0e685c-e2d3-436f-89bc-9d9fa3141b61)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[00:24:08.682]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:24:05.4122339Z);\\nlet _endTime = datetime(2026-08-25T00:24:05.4122344Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:08.6621602Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f\\\",\\r\\n            \\\"activityId\\\": \\\"ed3e04f4-6a49-4e81-9b00-d4558d9cddcc\\\",\\r\\n            \\\"subActivityId\\\": \\\"4cad9bba-6029-42b0-a77f-5c14bc4e8a84\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"58dfa59a-bdfb-430f-af23-104aaad94cc9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f ARID=ed3e04f4-6a49-4e81-9b00-d4558d9cddcc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/58dfa59a-bdfb-430f-af23-104aaad94cc9 \\u003e DN.FE.ExecuteQuery/4cad9bba-6029-42b0-a77f-5c14bc4e8a84)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:24:08.778]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:24:05.4122339Z);\\nlet _endTime = datetime(2026-08-25T00:24:05.4122344Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:08.6621602Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f\\\",\\r\\n            \\\"activityId\\\": \\\"ed3e04f4-6a49-4e81-9b00-d4558d9cddcc\\\",\\r\\n            \\\"subActivityId\\\": \\\"4cad9bba-6029-42b0-a77f-5c14bc4e8a84\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"58dfa59a-bdfb-430f-af23-104aaad94cc9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f ARID=ed3e04f4-6a49-4e81-9b00-d4558d9cddcc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/58dfa59a-bdfb-430f-af23-104aaad94cc9 \\u003e DN.FE.ExecuteQuery/4cad9bba-6029-42b0-a77f-5c14bc4e8a84)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00083c410\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[00:23:58.357]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:55.0351655Z);\\nlet _endTime = datetime(2026-08-25T00:23:55.0351661Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:23:58.3330999Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b\\\",\\r\\n            \\\"activityId\\\": \\\"442abb20-5490-45f5-ba70-3bca3bed0639\\\",\\r\\n            \\\"subActivityId\\\": \\\"b9132e8a-1c55-4f09-8982-f344852f0eee\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"99a04993-dace-423f-a879-2fbeee2a7b84\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b ARID=442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.Http.CallContext/442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.ExecuteQuery/99a04993-dace-423f-a879-2fbeee2a7b84 \\u003e DN.FE.ExecuteQuery/b9132e8a-1c55-4f09-8982-f344852f0eee)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:23:58.411]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:55.0351655Z);\\nlet _endTime = datetime(2026-08-25T00:23:55.0351661Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:23:58.3330999Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b\\\",\\r\\n            \\\"activityId\\\": \\\"442abb20-5490-45f5-ba70-3bca3bed0639\\\",\\r\\n            \\\"subActivityId\\\": \\\"b9132e8a-1c55-4f09-8982-f344852f0eee\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"99a04993-dace-423f-a879-2fbeee2a7b84\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b ARID=442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.Http.CallContext/442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.ExecuteQuery/99a04993-dace-423f-a879-2fbeee2a7b84 \\u003e DN.FE.ExecuteQuery/b9132e8a-1c55-4f09-8982-f344852f0eee)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[00:24:01.787]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:58.4821435Z);\\nlet _endTime = datetime(2026-08-25T00:23:58.4821442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:01.7618170Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5\\\",\\r\\n            \\\"activityId\\\": \\\"d2adae1e-7609-4016-b166-5a8ff8be55d8\\\",\\r\\n            \\\"subActivityId\\\": \\\"7c0e685c-e2d3-436f-89bc-9d9fa3141b61\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"8cf560a3-f257-4d9d-900b-aaf46185fe8b\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5 ARID=d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.Http.CallContext/d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.ExecuteQuery/8cf560a3-f257-4d9d-900b-aaf46185fe8b \\u003e DN.FE.ExecuteQuery/7c0e685c-e2d3-436f-89bc-9d9fa3141b61)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:24:01.845]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:58.4821435Z);\\nlet _endTime = datetime(2026-08-25T00:23:58.4821442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:01.7618170Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5\\\",\\r\\n            \\\"activityId\\\": \\\"d2adae1e-7609-4016-b166-5a8ff8be55d8\\\",\\r\\n            \\\"subActivityId\\\": \\\"7c0e685c-e2d3-436f-89bc-9d9fa3141b61\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"8cf560a3-f257-4d9d-900b-aaf46185fe8b\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5 ARID=d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.Http.CallContext/d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.ExecuteQuery/8cf560a3-f257-4d9d-900b-aaf46185fe8b \\u003e DN.FE.ExecuteQuery/7c0e685c-e2d3-436f-89bc-9d9fa3141b61)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[00:24:08.682]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:24:05.4122339Z);\\nlet _endTime = datetime(2026-08-25T00:24:05.4122344Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:08.6621602Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f\\\",\\r\\n            \\\"activityId\\\": \\\"ed3e04f4-6a49-4e81-9b00-d4558d9cddcc\\\",\\r\\n            \\\"subActivityId\\\": \\\"4cad9bba-6029-42b0-a77f-5c14bc4e8a84\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"58dfa59a-bdfb-430f-af23-104aaad94cc9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f ARID=ed3e04f4-6a49-4e81-9b00-d4558d9cddcc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/58dfa59a-bdfb-430f-af23-104aaad94cc9 \\u003e DN.FE.ExecuteQuery/4cad9bba-6029-42b0-a77f-5c14bc4e8a84)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:24:08.778]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:24:05.4122339Z);\\nlet _endTime = datetime(2026-08-25T00:24:05.4122344Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:08.6621602Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f\\\",\\r\\n            \\\"activityId\\\": \\\"ed3e04f4-6a49-4e81-9b00-d4558d9cddcc\\\",\\r\\n            \\\"subActivityId\\\": \\\"4cad9bba-6029-42b0-a77f-5c14bc4e8a84\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"58dfa59a-bdfb-430f-af23-104aaad94cc9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f ARID=ed3e04f4-6a49-4e81-9b00-d4558d9cddcc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/58dfa59a-bdfb-430f-af23-104aaad94cc9 \\u003e DN.FE.ExecuteQuery/4cad9bba-6029-42b0-a77f-5c14bc4e8a84)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000626a40\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[00:14:56.798]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:53.4068554Z);\\nlet _endTime = datetime(2026-08-25T00:14:53.4068559Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:14:56.7779164Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d\\\",\\r\\n            \\\"activityId\\\": \\\"697030e7-6169-48d4-afab-e55af5423283\\\",\\r\\n            \\\"subActivityId\\\": \\\"7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"e5629797-dc74-431c-89f2-26cc2ad935e1\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d ARID=697030e7-6169-48d4-afab-e55af5423283 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5629797-dc74-431c-89f2-26cc2ad935e1 \\u003e DN.FE.ExecuteQuery/7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:14:56.846]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:53.4068554Z);\\nlet _endTime = datetime(2026-08-25T00:14:53.4068559Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:14:56.7779164Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d\\\",\\r\\n            \\\"activityId\\\": \\\"697030e7-6169-48d4-afab-e55af5423283\\\",\\r\\n            \\\"subActivityId\\\": \\\"7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"e5629797-dc74-431c-89f2-26cc2ad935e1\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d ARID=697030e7-6169-48d4-afab-e55af5423283 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5629797-dc74-431c-89f2-26cc2ad935e1 \\u003e DN.FE.ExecuteQuery/7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[00:15:00.265]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:56.9537642Z);\\nlet _endTime = datetime(2026-08-25T00:14:56.9537648Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:00.2458657Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130\\\",\\r\\n            \\\"activityId\\\": \\\"60865211-a5af-4242-8bd8-14b7a1f199ba\\\",\\r\\n            \\\"subActivityId\\\": \\\"865c926e-620f-4107-9fd8-43ac9ee494b3\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"70a56a06-7a14-4b6c-b645-990ed51e93ae\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130 ARID=60865211-a5af-4242-8bd8-14b7a1f199ba \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/70a56a06-7a14-4b6c-b645-990ed51e93ae \\u003e DN.FE.ExecuteQuery/865c926e-620f-4107-9fd8-43ac9ee494b3)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:15:00.327]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:56.9537642Z);\\nlet _endTime = datetime(2026-08-25T00:14:56.9537648Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:00.2458657Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130\\\",\\r\\n            \\\"activityId\\\": \\\"60865211-a5af-4242-8bd8-14b7a1f199ba\\\",\\r\\n            \\\"subActivityId\\\": \\\"865c926e-620f-4107-9fd8-43ac9ee494b3\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"70a56a06-7a14-4b6c-b645-990ed51e93ae\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130 ARID=60865211-a5af-4242-8bd8-14b7a1f199ba \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/70a56a06-7a14-4b6c-b645-990ed51e93ae \\u003e DN.FE.ExecuteQuery/865c926e-620f-4107-9fd8-43ac9ee494b3)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[00:15:08.703]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:04.4342198Z);\\nlet _endTime = datetime(2026-08-25T00:15:04.4342235Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:08.6768100Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199\\\",\\r\\n            \\\"activityId\\\": \\\"a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19\\\",\\r\\n            \\\"subActivityId\\\": \\\"dda68eb2-c740-498f-b62e-c74a25b621f7\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"4d4ca23c-dcca-4b88-9136-446102508ccb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199 ARID=a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.Http.CallContext/a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.ExecuteQuery/4d4ca23c-dcca-4b88-9136-446102508ccb \\u003e DN.FE.ExecuteQuery/dda68eb2-c740-498f-b62e-c74a25b621f7)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:15:08.705]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"context canceled\",\n      \"name\": \"cosmosResourceSnapshots\"\n    }�[0m\n    �[37m[00:15:09.096]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:04.4342198Z);\\nlet _endTime = datetime(2026-08-25T00:15:04.4342235Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:08.6768100Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199\\\",\\r\\n            \\\"activityId\\\": \\\"a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19\\\",\\r\\n            \\\"subActivityId\\\": \\\"dda68eb2-c740-498f-b62e-c74a25b621f7\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"4d4ca23c-dcca-4b88-9136-446102508ccb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199 ARID=a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.Http.CallContext/a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.ExecuteQuery/4d4ca23c-dcca-4b88-9136-446102508ccb \\u003e DN.FE.ExecuteQuery/dda68eb2-c740-498f-b62e-c74a25b621f7)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000626a40\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[00:14:56.798]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:53.4068554Z);\\nlet _endTime = datetime(2026-08-25T00:14:53.4068559Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:14:56.7779164Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d\\\",\\r\\n            \\\"activityId\\\": \\\"697030e7-6169-48d4-afab-e55af5423283\\\",\\r\\n            \\\"subActivityId\\\": \\\"7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"e5629797-dc74-431c-89f2-26cc2ad935e1\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d ARID=697030e7-6169-48d4-afab-e55af5423283 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5629797-dc74-431c-89f2-26cc2ad935e1 \\u003e DN.FE.ExecuteQuery/7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:14:56.846]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:53.4068554Z);\\nlet _endTime = datetime(2026-08-25T00:14:53.4068559Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:14:56.7779164Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d\\\",\\r\\n            \\\"activityId\\\": \\\"697030e7-6169-48d4-afab-e55af5423283\\\",\\r\\n            \\\"subActivityId\\\": \\\"7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"e5629797-dc74-431c-89f2-26cc2ad935e1\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d ARID=697030e7-6169-48d4-afab-e55af5423283 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5629797-dc74-431c-89f2-26cc2ad935e1 \\u003e DN.FE.ExecuteQuery/7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[00:15:00.265]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:56.9537642Z);\\nlet _endTime = datetime(2026-08-25T00:14:56.9537648Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:00.2458657Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130\\\",\\r\\n            \\\"activityId\\\": \\\"60865211-a5af-4242-8bd8-14b7a1f199ba\\\",\\r\\n            \\\"subActivityId\\\": \\\"865c926e-620f-4107-9fd8-43ac9ee494b3\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"70a56a06-7a14-4b6c-b645-990ed51e93ae\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130 ARID=60865211-a5af-4242-8bd8-14b7a1f199ba \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/70a56a06-7a14-4b6c-b645-990ed51e93ae \\u003e DN.FE.ExecuteQuery/865c926e-620f-4107-9fd8-43ac9ee494b3)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:15:00.327]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:56.9537642Z);\\nlet _endTime = datetime(2026-08-25T00:14:56.9537648Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:00.2458657Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130\\\",\\r\\n            \\\"activityId\\\": \\\"60865211-a5af-4242-8bd8-14b7a1f199ba\\\",\\r\\n            \\\"subActivityId\\\": \\\"865c926e-620f-4107-9fd8-43ac9ee494b3\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"70a56a06-7a14-4b6c-b645-990ed51e93ae\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130 ARID=60865211-a5af-4242-8bd8-14b7a1f199ba \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/70a56a06-7a14-4b6c-b645-990ed51e93ae \\u003e DN.FE.ExecuteQuery/865c926e-620f-4107-9fd8-43ac9ee494b3)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[00:15:08.703]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:04.4342198Z);\\nlet _endTime = datetime(2026-08-25T00:15:04.4342235Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:08.6768100Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199\\\",\\r\\n            \\\"activityId\\\": \\\"a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19\\\",\\r\\n            \\\"subActivityId\\\": \\\"dda68eb2-c740-498f-b62e-c74a25b621f7\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"4d4ca23c-dcca-4b88-9136-446102508ccb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199 ARID=a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.Http.CallContext/a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.ExecuteQuery/4d4ca23c-dcca-4b88-9136-446102508ccb \\u003e DN.FE.ExecuteQuery/dda68eb2-c740-498f-b62e-c74a25b621f7)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:15:08.705]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"context canceled\",\n      \"name\": \"cosmosResourceSnapshots\"\n    }�[0m\n    �[37m[00:15:09.096]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:04.4342198Z);\\nlet _endTime = datetime(2026-08-25T00:15:04.4342235Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:08.6768100Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199\\\",\\r\\n            \\\"activityId\\\": \\\"a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19\\\",\\r\\n            \\\"subActivityId\\\": \\\"dda68eb2-c740-498f-b62e-c74a25b621f7\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"4d4ca23c-dcca-4b88-9136-446102508ccb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199 ARID=a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.Http.CallContext/a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.ExecuteQuery/4d4ca23c-dcca-4b88-9136-446102508ccb \\u003e DN.FE.ExecuteQuery/dda68eb2-c740-498f-b62e-c74a25b621f7)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","signature_id":"93cd087eb0fc0ac2b0c79ba3bc6c6181f71cba9db1ac3eaefd4c1d8152fbb877","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092031971118026752","occurred_at":"2026-08-24T23:30:58Z","signature_id":"176c7f8e0e9a6fe4e5a12f9055659eede136ab94bf2bc621167d758c6dc1828c","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092030482601480192","occurred_at":"2026-08-24T23:25:09Z","signature_id":"a4d769910ea57a048ae6030e8851eaac93f508a9d62968944253d616d8fe88ed","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092025885698822144","occurred_at":"2026-08-24T23:06:53Z","signature_id":"9eb8d09c21becab11e4df2e2486fbadfcfe682a12484ff5957cc08b54e0cd902","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","signature_id":"a99ad7aa848b9f85d32a71fb35dfd49c3d8270fdc830d4478b6af42e4702a425","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","signature_id":"0d5221b00aeb7bfdca1da1581a9d7578e9d19b098899c13f87a46837f2394f22","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092014812023230464","occurred_at":"2026-08-24T22:22:53Z","signature_id":"c5fd89f1c41b331bfa0c3c5ff918b5ee94282a73960beb6c8680dfabd0d92aba","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092003732647579648","occurred_at":"2026-08-24T21:38:47Z","signature_id":"c3920cca5c3b22e9ea0a8df63d7fd82dff370a2c49fd2a56d2b8612e6611e8cd","pr_number":6088,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091996785936437248","occurred_at":"2026-08-24T21:11:15Z","signature_id":"c79de5565353120aa973caf9da2e4e0043d28fd25ac56a7cd204827ac9e612e5","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091995273302642688","occurred_at":"2026-08-24T21:05:15Z","signature_id":"b19e1030a263db434fc5938127062d3add16c8d3a56319f83b1663785d30bba9","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"98f2c85bfff864b394f0a6540743e0876e4af34e9f265ec3b983e45e8ef64a57","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"fa1482e0c10fe89a03896ac5f14bc5ff11cbc2b5c024fd993535afa6403d322b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985911028912128","occurred_at":"2026-08-24T20:28:03Z","signature_id":"43910364f48f66523f1f9d08a417492faf101b0ab2cb444442ab1a553439c6b5","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985362346840064","occurred_at":"2026-08-24T20:25:41Z","signature_id":"176cc4504c2f048683d4e9fd8346ddda26590ba46f5fcff44ce3756a85f5565e","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"913731872ab076d2435270c0fedcc39b2d88a5ff36047161ca4eabe2b648b67d","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091978207036182528","occurred_at":"2026-08-24T19:57:24Z","signature_id":"f663b43ed0edc7e9f34c5b7d6058cf02871c4b2cbf7c46200eb2800bb395c1fc","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977503907254272","occurred_at":"2026-08-24T19:54:38Z","signature_id":"0b9a16666408f3f71c77e5a20e559de4e428d06efaede29056c5fdec679a0549","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977495384428544","occurred_at":"2026-08-24T19:54:36Z","signature_id":"07fcf787e9667bb548a1ae4e2aa892f82683b85aa913615ccc3c55f328e44f13","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977485536202752","occurred_at":"2026-08-24T19:54:34Z","signature_id":"edf4f7b0d5436e65624e47252a3c9819a947e53aeb06b94097e54f965d358ffa","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","signature_id":"69234d1190be6bae030c9f0be83f507930b8aba09ce9643964f39fe12af78ce2","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963250454302720","occurred_at":"2026-08-24T18:58:00Z","signature_id":"1bd7b81142bf5c01b0963c412981b707f78cb8184adf320516339e4aa3a3ce9f","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963205222928384","occurred_at":"2026-08-24T18:57:49Z","signature_id":"9bfada2b49b3fc5245efb1ec3e602fd604d8132a904f08045a044c170e893680","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"c742492599a708bf77717b7e4360828e91602b79b9e3f94c759bffe66f0abf90","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091960563839012864","occurred_at":"2026-08-24T18:47:19Z","signature_id":"ec35c9667d01d0fca7a963152324f60489cc7cd6ddc292c3798abbfb8aa51092","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091956227226472448","occurred_at":"2026-08-24T18:30:02Z","signature_id":"335052cc293f1dcfc9068c799f022fd529e07859d0cea778ae29ff1ed7adcc44","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091950955841982464","occurred_at":"2026-08-24T18:09:09Z","signature_id":"c0cebe2544e94f85956b2aa4e6cb37d39972915968a1824b87954f1d8f2b0278","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091948175697907712","occurred_at":"2026-08-24T17:58:06Z","signature_id":"2cfd9e8dd04e05dbf25d7770f4e1349a8d7b20bfb77b1b160b57373b94af3940","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091942092610736128","occurred_at":"2026-08-24T17:33:55Z","signature_id":"41ffa53c8fc1675a8e8f11ea4fff86270a85dca8d99ed458236b7319ea0aa4d4","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"dec2f42fbc616357b928df0a2e19436696b2e782470cc0447de0e33d58d63917","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932257060130816","occurred_at":"2026-08-24T16:54:39Z","signature_id":"cff96711d2e8bb5b9f48911a18c760f7b843abef0ee0b1364775c2b0b5f58a49","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932359686361088","occurred_at":"2026-08-24T16:54:34Z","signature_id":"5fa5e4ca1b4d0f869a74c660939188cb0fcaacbc16b74da34e727232c29abbff","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"6ff77bfa149b82860b0429a7dfe5ab2b71a58982ddc24dc6fbf5996e7d1dca8d","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"1ae3c1ffd864572007c734910a8d27ab7b2bea356002ed663c85c3e98b525248","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"9865c7135fd03ceb69a76994157b6614c9b64d8eebb69b3b00180111a55f75a4","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091926769790619648","occurred_at":"2026-08-24T16:21:04Z","signature_id":"40f2371fc4b699bfbeed8a52087620105728edb5da70261d5e9bd7df99b13e6a","pr_number":6639,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":31,"runs_affected":20,"failed_runs":20,"run_impact_percent":19.41747572815534,"anchor_occurrences":31,"after_last_push_count":16,"also_in":["INT"],"trend_counts":[18,2,3,0,4,0,31],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":53,"prior_last_seen_at":"2026-08-22T18:55:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-h74snzg5tpbh/cluster-candidate-5-0-gp9ztx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014a05c0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-gp9ztx: failed waiting for cluster=\"cluster-candidate-5-0-gp9ztx\" in resourcegroup=\"rg-candidate-5-0-h74snzg5tpbh\" to finish creating: GET https://rp.j1843200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de4739f4-eebd-4fe2-bf80-3bf6d57ff617\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de4739f4-eebd-4fe2-bf80-3bf6d57ff617\",\n      \"name\": \"de4739f4-eebd-4fe2-bf80-3bf6d57ff617\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T00:23:47.993778746Z\",\n      \"endTime\": \"2026-08-25T00:42:49.076566651Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-h74snzg5tpbh/cluster-candidate-5-0-gp9ztx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014a05c0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-gp9ztx: failed waiting for cluster=\"cluster-candidate-5-0-gp9ztx\" in resourcegroup=\"rg-candidate-5-0-h74snzg5tpbh\" to finish creating: GET https://rp.j1843200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de4739f4-eebd-4fe2-bf80-3bf6d57ff617\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de4739f4-eebd-4fe2-bf80-3bf6d57ff617\",\n      \"name\": \"de4739f4-eebd-4fe2-bf80-3bf6d57ff617\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T00:23:47.993778746Z\",\n      \"endTime\": \"2026-08-25T00:42:49.076566651Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-x45vwlgpgvzk/cluster-candidate-4-21-rztpw4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3e6a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-rztpw4: failed waiting for cluster=\"cluster-candidate-4-21-rztpw4\" in resourcegroup=\"rg-candidate-4-21-x45vwlgpgvzk\" to finish creating: GET https://rp.j4709760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e986c420-0503-4186-9a01-bad1f92a005c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e986c420-0503-4186-9a01-bad1f92a005c\",\n      \"name\": \"e986c420-0503-4186-9a01-bad1f92a005c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:37:07.202301694Z\",\n      \"endTime\": \"2026-08-24T23:56:10.677598487Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-x45vwlgpgvzk/cluster-candidate-4-21-rztpw4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3e6a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-rztpw4: failed waiting for cluster=\"cluster-candidate-4-21-rztpw4\" in resourcegroup=\"rg-candidate-4-21-x45vwlgpgvzk\" to finish creating: GET https://rp.j4709760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e986c420-0503-4186-9a01-bad1f92a005c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e986c420-0503-4186-9a01-bad1f92a005c\",\n      \"name\": \"e986c420-0503-4186-9a01-bad1f92a005c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:37:07.202301694Z\",\n      \"endTime\": \"2026-08-24T23:56:10.677598487Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d48840\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-mglhphvdtjhd\" to finish creating: GET https://rp.j6339200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\",\n      \"name\": \"1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:29:18.405302537Z\",\n      \"endTime\": \"2026-08-24T23:48:26.458954406Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d48840\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-mglhphvdtjhd\" to finish creating: GET https://rp.j6339200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\",\n      \"name\": \"1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:29:18.405302537Z\",\n      \"endTime\": \"2026-08-24T23:48:26.458954406Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","signature_id":"54a530c9876fc008fec6ac6899f443f3e3f5628a64422efb1e6064e8186af6e1","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","signature_id":"e6a9db1c5a3cafd8f545d8476a1e6b49dd9166a98aa20d59709936d40b1c52c0","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092019318316339200","occurred_at":"2026-08-24T22:40:47Z","signature_id":"a6a8523a32b9fba91a90fbd04abe90465e11b86cdf06949b929a37a85e4f40ad","pr_number":6663,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092019318316339200","occurred_at":"2026-08-24T22:40:47Z","signature_id":"e6285c4c12e9e0c85b5ecc25058de40b9f05b37d1a08c8a8c4a66108766b8ee0","pr_number":6663,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","signature_id":"4549c28d9f346a4101b2d199c56363a0a36787b6952f32440182d5210d9903d4","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","signature_id":"916b71cbe1c0282bebb7706c18cf185456f94291c6c7059f8d673bee6920e761","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"49911eb59aa73a3d7ebf02bdb4e56de7a616897c72e3ac7abe420c0c7a47e40e","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"6487190a1499ee6eb043aa40a245add60c110f31922a104a2b7ccae5c75948f8","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"7cc1252e9b8fe966bbe77836b76cbb84f92a735b546bfadc6681cd7399872b22","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"a9397a1f0aaba5d8ad982fe383aca1267736bcbd0d40f55741f427e85ee8b6d8","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"c5d8b0e5c56e2c31565dc9e53627bd3252420239c844111c2d9db849f9e9c51c","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"deae4888a32973190e55fe6f1f2ac8a8941dd6efae21aa17eaa50cc4448f5b70","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"7451289a87442818c6daa03144d0fb9efebafa88d3687d31f99e8439ca741e97","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","signature_id":"7830c998ab91a0251740ef30429a36f78ee908a1781d172a1f172ebef622c56d","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963250454302720","occurred_at":"2026-08-24T18:58:00Z","signature_id":"75d7cd315bc777c1079b9374c7a2b4aabd61d42a04817f2523f7ea7dab1326a9","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"2021d27012220f14a5eaeb1e574cf0e0cf00596ef715d56d7ab033d1204d0ed5","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"dea6785389a59c2a975d6ddfe69a85996e881ce704015e7a9c41548aeaca0017","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"19ba2327e177a466c74ef48f9d128288b644237f0c8199be7309ae0bf40f5a82","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932257060130816","occurred_at":"2026-08-24T16:54:39Z","signature_id":"70c12268048fb1a990e1fcfe48d8d53530e6791148ebcbe91068d54355e50247","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"688dcfad091ac3bb942f27fcd5e2e3d6471f49c9042338c1c965ab6c339ae7e6","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"d1005ca6dc326e3c744c86ae1e9fb1086de2e12e4a6351183145527b31e5eccf","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"db07fb00cb7c1dc1d49041e7a151eab39b4fee2bc9901dd00a1fc182625826d2","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"40b147b3a09185f7fbbd5416adf8d0e158c67796d5387c2aa465c0a43fd1dc30","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"8d53ed6b1258502dfa6cecd4e6049f72772418b7686ae5c9e29c6adc65e73c79","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"fd285a3626d258dcb1ba2d99b9eb4762dc5e5af32183bf400bd799c872dd2fda","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"94f9ba782dfa48d054a8eb8de04c3bde301c62297127cae4d03aeb5f65b1b3b4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"e6714abbbcfe1d54c45ec38583b6367bc1667d055485de6ce77544f68acc9679","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"ccd1c15e8b2737a8cbb30d07ada8f1b6f0748afd9d43b748ec4128f7a61ce044","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879954055696384","occurred_at":"2026-08-24T13:27:00Z","signature_id":"b66537b24ffd588d399e5e7cc810bf3a188e5e0dce669b0b16956a016a18a10c","pr_number":6429,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879927736438784","occurred_at":"2026-08-24T13:26:52Z","signature_id":"6a060f99c27004cf01810556d4b6e1126f6aeee13d0cc7b3c046327b6e9bcc0d","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"25baffda5fc6f88c385cae06e401a6f4815895c798a9d1e087a18364db777535","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":28,"runs_affected":28,"failed_runs":28,"run_impact_percent":27.184466019417474,"anchor_occurrences":28,"after_last_push_count":7,"trend_counts":[0,0,0,6,0,0,28],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":34,"prior_last_seen_at":"2026-08-21T03:51:03Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":6},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":4},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":18}],"full_error_samples":["time=2026-08-24T13:26:56.540Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:26:58.762Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T13:27:00.485Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=abeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8084480-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fabeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969\ntime=2026-08-24T13:35:32.482Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/abeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969/operationStatuses/08584140288659502671\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/cluster-kyv757fphd2fm\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j8084480-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-24652102-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-24652102-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-24T13:31:41.774Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:31:42.863Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T13:31:44.020Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=cf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5557248-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5\ntime=2026-08-24T13:39:15.842Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/cf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5/operationStatuses/08584140285820210806\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/cluster-ce5kkcjcnxbu2\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5557248-mgmt-2/agentPools/infra1\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-20013346-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-20013346-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-24T13:26:28.921Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:26:30.957Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T13:26:32.602Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2132480-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848\ntime=2026-08-24T13:34:04.585Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848/operationStatuses/08584140288938416315\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/cluster-d54rkmqgu5xlq\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2132480-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-42034125-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-42034125-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091873075288084480","occurred_at":"2026-08-24T12:59:40Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872721045557248","occurred_at":"2026-08-24T12:58:16Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872693082132480","occurred_at":"2026-08-24T12:58:09Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872666397970432","occurred_at":"2026-08-24T12:58:03Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091868107806085120","occurred_at":"2026-08-24T12:39:55Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6585,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091867251824136192","occurred_at":"2026-08-24T12:36:32Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6585,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091856057667162112","occurred_at":"2026-08-24T11:52:03Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6585,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091852685220253696","occurred_at":"2026-08-24T11:34:18Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6424,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844440330080256","occurred_at":"2026-08-24T11:05:53Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844377495212032","occurred_at":"2026-08-24T11:05:38Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844284889174016","occurred_at":"2026-08-24T11:05:16Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091840235057975296","occurred_at":"2026-08-24T10:49:11Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091827867787726848","occurred_at":"2026-08-24T10:00:02Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091827277728845824","occurred_at":"2026-08-24T09:57:41Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6429,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091823405572558848","occurred_at":"2026-08-24T09:42:18Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091816375096971264","occurred_at":"2026-08-24T09:14:22Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6429,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091814034880860160","occurred_at":"2026-08-24T09:05:04Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6517,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091812979992432640","occurred_at":"2026-08-24T09:00:53Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091803125458407424","occurred_at":"2026-08-24T08:21:34Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091801419551084544","occurred_at":"2026-08-24T08:14:51Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091795137024757760","occurred_at":"2026-08-24T07:49:58Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091792609700745216","occurred_at":"2026-08-24T07:39:56Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6278,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6656/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091776267920609280","occurred_at":"2026-08-24T06:35:00Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6656,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091774664165560320","occurred_at":"2026-08-24T06:28:37Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091750724990930944","occurred_at":"2026-08-24T04:53:30Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091747201922371584","occurred_at":"2026-08-24T04:39:30Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091734160543584256","occurred_at":"2026-08-24T03:47:41Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091733541242015744","occurred_at":"2026-08-24T03:45:13Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6646,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":14,"runs_affected":14,"failed_runs":14,"run_impact_percent":13.592233009708737,"anchor_occurrences":14,"after_last_push_count":70,"trend_counts":[14,9,14,12,2,0,14],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":348,"prior_last_seen_at":"2026-08-22T05:45:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":14}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T00:35:25Z\n  Ended:       2026-08-25T00:56:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1843200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd9551d19b6bblvufm3l3a421htavmi-a4k0z5e8x0e3t0f\", pod=\"router-597d99bc98-tlmg9\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sd9551d19b6bblvufm3l3a421htavmi-a4k0z5e8x0e3t0f/router-597d99bc98-tlmg9 has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T23:47:43Z\n  Ended:       2026-08-25T00:09:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4709760-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd8f93h71j1qrcah2q03eja41gojmif-i6w1a1c6f6m8f7l\", pod=\"router-7f97cf9c5-phjp6\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sd8f93h71j1qrcah2q03eja41gojmif-i6w1a1c6f6m8f7l/router-7f97cf9c5-phjp6 has been in a non-ready state for longer than 5 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T22:04:58Z\n  Ended:       2026-08-24T22:16:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9409536-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd6tku01d4e5c0s0k2ocfdtf91cpr8l-d3q6r6q7q0m9f7w\", pod=\"router-57846fcf5b-n5mlw\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sd6tku01d4e5c0s0k2ocfdtf91cpr8l-d3q6r6q7q0m9f7w/router-57846fcf5b-n5mlw has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T22:04:58Z\n  Ended:       2026-08-24T22:16:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9409536-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd6td57g5fetk5hnhg543eg0jfi9lrt-j5s2f3c0f5x7m8z\", pod=\"router-7586ffccfd-4mvlg\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sd6td57g5fetk5hnhg543eg0jfi9lrt-j5s2f3c0f5x7m8z/router-7586ffccfd-4mvlg has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","signature_id":"2f1d9b2b8857e75d809a23da90c2e32bd0a70345c25f6862fc7292ac5c796db3","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","signature_id":"f8ca631bf4ba3de6a823751843028c53bf8f63552f781736fc0f06124e42925e","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"762c894801e83a205ad61ad7c1a1171fcbbd7db65d82ac38613bccfd72e8ae0d","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"5a62ef9a7062e35c1e10c6c46839368f20045e53d18429e0c93ad1b8118f4ed4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"05c28635da15b6abb7fa52304cc82cedf7070115a67070cb7058c7bc17ceed55","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","signature_id":"d9b2b2f48eba30a813d1493c2cf9d779d74129e8e68690c29d48df3b77d5680a","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"84936854ba2393da3d6d4a90cbe465eac2bfa99410eb5ae262e1869d118d0d44","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"d6263c32bd22604b284f6288625ad3fcd30c678787956b94fe5bf7e9a8e0f6cc","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"ddb17f85d08dfa0f0d60af8e146b060c3f080bb3cb873c1fd946241da0c96c74","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091920407983558656","occurred_at":"2026-08-24T16:06:36Z","signature_id":"c52899aff7bf73e0f12d284cdf7344825014b3fd387fb2f270a8871271bc7acb","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"32b177871d4669aa26bebdb303a13d4eeafa32be58fe0c29eecd2b7355b315ff","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"d1d5a7d0ff60a9161bf6b8e8b3cd897227641812139ecab6ded5ff874a074bf7","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"41f9fa028fb65313589b4f31abf40b7c37b7c57dd66b9ac548b1148f02514739","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"d55af07fa2830b9ad7b195c673e4e375ec2e39f3e870f6e6e4139ca2af8abfa0","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":10,"runs_affected":4,"failed_runs":4,"run_impact_percent":3.883495145631068,"anchor_occurrences":10,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,10],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-22lvs576ptrb/cluster-candidate-4-20-qdcf69 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001190ba0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-qdcf69: failed waiting for cluster=\"cluster-candidate-4-20-qdcf69\" in resourcegroup=\"rg-candidate-4-20-22lvs576ptrb\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc1dd563-6c7e-4d20-8fce-99e12ec22770\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc1dd563-6c7e-4d20-8fce-99e12ec22770\",\n      \"name\": \"dc1dd563-6c7e-4d20-8fce-99e12ec22770\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:51:04.562343833Z\",\n      \"endTime\": \"2026-08-24T22:10:06.652345811Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-22lvs576ptrb/cluster-candidate-4-20-qdcf69 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001190ba0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-qdcf69: failed waiting for cluster=\"cluster-candidate-4-20-qdcf69\" in resourcegroup=\"rg-candidate-4-20-22lvs576ptrb\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc1dd563-6c7e-4d20-8fce-99e12ec22770\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc1dd563-6c7e-4d20-8fce-99e12ec22770\",\n      \"name\": \"dc1dd563-6c7e-4d20-8fce-99e12ec22770\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:51:04.562343833Z\",\n      \"endTime\": \"2026-08-24T22:10:06.652345811Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e33220\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-ndmwcdqgk685\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a097060e-673b-40de-a82a-aa623235d88d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a097060e-673b-40de-a82a-aa623235d88d\",\n      \"name\": \"a097060e-673b-40de-a82a-aa623235d88d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:51:06.650394368Z\",\n      \"endTime\": \"2026-08-24T22:10:06.638990375Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e33220\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-ndmwcdqgk685\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a097060e-673b-40de-a82a-aa623235d88d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a097060e-673b-40de-a82a-aa623235d88d\",\n      \"name\": \"a097060e-673b-40de-a82a-aa623235d88d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:51:06.650394368Z\",\n      \"endTime\": \"2026-08-24T22:10:06.638990375Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:148]: failed to create HCP cluster np-version-upgrade-cluster-vkwj8z with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bca9c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-vkwj8z: failed waiting for cluster=\"np-version-upgrade-cluster-vkwj8z\" in resourcegroup=\"rg-np-version-upgrade-vkwj8z-pvn2kjtfhwm4\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ca593c6-172c-442e-bdb4-2b96ae721c04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ca593c6-172c-442e-bdb4-2b96ae721c04\",\n      \"name\": \"9ca593c6-172c-442e-bdb4-2b96ae721c04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:29:02.41159338Z\",\n      \"endTime\": \"2026-08-24T21:48:06.346460728Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:148]: failed to create HCP cluster np-version-upgrade-cluster-vkwj8z with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bca9c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-vkwj8z: failed waiting for cluster=\"np-version-upgrade-cluster-vkwj8z\" in resourcegroup=\"rg-np-version-upgrade-vkwj8z-pvn2kjtfhwm4\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ca593c6-172c-442e-bdb4-2b96ae721c04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ca593c6-172c-442e-bdb4-2b96ae721c04\",\n      \"name\": \"9ca593c6-172c-442e-bdb4-2b96ae721c04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:29:02.41159338Z\",\n      \"endTime\": \"2026-08-24T21:48:06.346460728Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"87055b770927de3226a00a72e6a1e8f425c4181b7305ba9fd94502a36df5fec7","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"e5919a5aa6e234b2c6b16539e4375d2ff9f7724d31d00cb9d8f2c6f3bd3b38ad","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"4bba4cf482e3fa39dfa2ce3e799d7eb2c477fcc7e6ad0b0350124af0d131c053","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"5a1d95e1ce2afb3cd1318374aea17f4a6e4f5d30ef03f9f5a0cc441e9043a771","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"28a8b869f35a9cd1794a8e197fc2255f15a11da6d73b85987e232efa1871d3dc","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"2cae48dc9a7ac77e5a8ac54a32e19e04d69dc07b4bbfdffac675fbb83525b59a","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"51d9d625d26042275bccf29b990ed16b6ab0c82bab381da48ae1943a65da776a","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"57e93beced19ad75ed8576f3f0efd935e4fd8431d2ed07db01840f4e2fefa928","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"9137a39c80626f35b65cbd5b530df9d8f5ceed8a3ed523d23d20fe02b5217e4f","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"0119ef26d501844a6f69a87b74e13b68dec77db9c3af742f0d6e0ca5c74e1853","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":5.825242718446602,"anchor_occurrences":6,"after_last_push_count":9,"trend_counts":[1,1,0,0,1,0,6],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":54,"prior_last_seen_at":"2026-08-22T04:44:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985911028912128","occurred_at":"2026-08-24T20:28:03Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"371e2dd52ef73c97161118e4d4efe23b9ace8422b8af743704455c3d7faec23b","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":3.883495145631068,"anchor_occurrences":4,"after_last_push_count":9,"trend_counts":[3,2,1,1,1,0,4],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":12,"prior_last_seen_at":"2026-08-22T00:10:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00123e240\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2xcnpkr48kxk\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3185d8c9-e246-4082-b170-898da28192d1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3185d8c9-e246-4082-b170-898da28192d1\",\n      \"name\": \"3185d8c9-e246-4082-b170-898da28192d1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:28:32.888311428Z\",\n      \"endTime\": \"2026-08-24T17:47:34.534131157Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00123e240\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2xcnpkr48kxk\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3185d8c9-e246-4082-b170-898da28192d1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3185d8c9-e246-4082-b170-898da28192d1\",\n      \"name\": \"3185d8c9-e246-4082-b170-898da28192d1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:28:32.888311428Z\",\n      \"endTime\": \"2026-08-24T17:47:34.534131157Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:726]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd0320\u003e: \n    failed to create HCP cluster np-dg-minor-4tds4q: failed waiting for cluster=\"np-dg-minor-4tds4q\" in resourcegroup=\"rg-np-dg-minor-4tds4q-6qtw9l2fwcdn\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/162a10b5-886b-40a8-a95b-30fc38ce9bc5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/162a10b5-886b-40a8-a95b-30fc38ce9bc5\",\n      \"name\": \"162a10b5-886b-40a8-a95b-30fc38ce9bc5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:41:41.593852701Z\",\n      \"endTime\": \"2026-08-24T16:00:49.106385261Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:726]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd0320\u003e: \n    failed to create HCP cluster np-dg-minor-4tds4q: failed waiting for cluster=\"np-dg-minor-4tds4q\" in resourcegroup=\"rg-np-dg-minor-4tds4q-6qtw9l2fwcdn\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/162a10b5-886b-40a8-a95b-30fc38ce9bc5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/162a10b5-886b-40a8-a95b-30fc38ce9bc5\",\n      \"name\": \"162a10b5-886b-40a8-a95b-30fc38ce9bc5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:41:41.593852701Z\",\n      \"endTime\": \"2026-08-24T16:00:49.106385261Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e96560\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-fn2m5nmfttf7\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\",\n      \"name\": \"eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:08.176674236Z\",\n      \"endTime\": \"2026-08-24T15:36:17.96458443Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e96560\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-fn2m5nmfttf7\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\",\n      \"name\": \"eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:08.176674236Z\",\n      \"endTime\": \"2026-08-24T15:36:17.96458443Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"e79e05105a6dd37d39aa319e6b82dc1c85c13b7322507d230499b9d98521e94c","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"7f0730f0d1c7e0eb7a035ce2e1a824a7fb99f16bf281917ec8e0ea592b939bdd","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"f45d8f4fc578e9bcbc89c15b0ea557b08d4323587837d33c561c269a0b539be9","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"f749c71c85c572aa2c57394be8a931e71c30eda717753e81fa6456d997c9c5fc","pr_number":6658,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":3.883495145631068,"anchor_occurrences":4,"after_last_push_count":108,"also_in":["STG"],"trend_counts":[3,2,0,27,5,0,4],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":148,"prior_last_seen_at":"2026-08-22T00:26:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:109]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000da71c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-wqnnn79bk4qc\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e635655f-c9ac-44e1-8232-3550a4c20673\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e635655f-c9ac-44e1-8232-3550a4c20673\",\n      \"name\": \"e635655f-c9ac-44e1-8232-3550a4c20673\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:28:57.457434277Z\",\n      \"endTime\": \"2026-08-24T21:48:06.423004565Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:109]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000da71c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-wqnnn79bk4qc\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e635655f-c9ac-44e1-8232-3550a4c20673\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e635655f-c9ac-44e1-8232-3550a4c20673\",\n      \"name\": \"e635655f-c9ac-44e1-8232-3550a4c20673\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:28:57.457434277Z\",\n      \"endTime\": \"2026-08-24T21:48:06.423004565Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:109]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa81a0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jhvt8ptm7fw8\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12485de2-db77-439f-89df-5bacb5d218b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12485de2-db77-439f-89df-5bacb5d218b2\",\n      \"name\": \"12485de2-db77-439f-89df-5bacb5d218b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:41:38.67903036Z\",\n      \"endTime\": \"2026-08-24T16:00:39.109568382Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:109]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa81a0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jhvt8ptm7fw8\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12485de2-db77-439f-89df-5bacb5d218b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12485de2-db77-439f-89df-5bacb5d218b2\",\n      \"name\": \"12485de2-db77-439f-89df-5bacb5d218b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:41:38.67903036Z\",\n      \"endTime\": \"2026-08-24T16:00:39.109568382Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e86920\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-7vpnvrbqnldf\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73cc6203-30af-4c53-9024-398221683c83\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73cc6203-30af-4c53-9024-398221683c83\",\n      \"name\": \"73cc6203-30af-4c53-9024-398221683c83\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:58.555796542Z\",\n      \"endTime\": \"2026-08-24T15:37:08.01693151Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e86920\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-7vpnvrbqnldf\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73cc6203-30af-4c53-9024-398221683c83\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73cc6203-30af-4c53-9024-398221683c83\",\n      \"name\": \"73cc6203-30af-4c53-9024-398221683c83\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:58.555796542Z\",\n      \"endTime\": \"2026-08-24T15:37:08.01693151Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"409a88ec07ceec5d40d5ae23f780d5c74cad626824811548eedaa155d1eba475","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"3605097c99e7a1623e2e3dd91c5b9641833e016d1c0c516d9b0275085f65863f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"4de409ed2fffcc46ddb557100c947bf3b037028f8c648ac70391763c58835df0","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"2cc87e67c265ba4f74a9a313e8f38a6f54437455cae5243f5e1f2cc3556cdfc9","pr_number":6658,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":2.912621359223301,"anchor_occurrences":3,"after_last_push_count":10,"trend_counts":[0,0,0,41,0,0,3],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":82,"prior_last_seen_at":"2026-08-21T20:12:08Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T00:10:04Z\n  Ended:       2026-08-25T00:34:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j8026752-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n  Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T22:22:12Z\n  Ended:       2026-08-24T22:46:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j2642688-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n  Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T19:54:46Z\n  Ended:       2026-08-24T20:17:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j4068096-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n  Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092031971118026752","occurred_at":"2026-08-24T23:30:58Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091995273302642688","occurred_at":"2026-08-24T21:05:15Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.941747572815534,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T09:11:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-fn4s2m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dea2e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-fn4s2m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dea2e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005da4a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005da4a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977495384428544","occurred_at":"2026-08-24T19:54:36Z","signature_id":"4a86fd68539d0c65cc2c794f0a3b117d62969a34ae1b535d87cc0cce6d88b528","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091811921836642304","occurred_at":"2026-08-24T08:56:40Z","signature_id":"67e0d9c3e541426a700c2b1097cab5407e75837246f0b2492c0fa4c512883fb9","pr_number":6618,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.941747572815534,"anchor_occurrences":2,"after_last_push_count":10,"trend_counts":[3,1,1,4,0,0,2],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":27,"prior_last_seen_at":"2026-08-21T16:05:36Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/management/output (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-08-24T18:57:37.594Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 description=\"Step output\\n  Kind: ARM\\n  Template: ./../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ./../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-24T18:57:37.979Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1\ntime=2026-08-24T18:57:38.350Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 deployment=cfd650f4557893b92396842785417ac4f900952cc40f35ef046b64198c66a837 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8797056-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcfd650f4557893b92396842785417ac4f900952cc40f35ef046b64198c66a837\ntime=2026-08-24T18:57:38.400Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8797056-mgmt-1/providers/Microsoft.Resources/deployments/cfd650f4557893b92396842785417ac4f900952cc40f35ef046b64198c66a837/operationStatuses/08584140090272578417\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentNotFound\\\",\\n    \\\"message\\\": \\\"Deployment 'cfd650f4557893b92396842785417ac4f900952cc40f35ef046b64198c66a837' could not be found.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-24T17:20:28.034Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 description=\"Step output\\n  Kind: ARM\\n  Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-24T17:20:28.426Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1\ntime=2026-08-24T17:20:28.817Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 deployment=3472b226dd1bf3f5c2c2fbd8f62937b0fe2f97cb4ca8a63168999f9ba0945d23 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2076032-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3472b226dd1bf3f5c2c2fbd8f62937b0fe2f97cb4ca8a63168999f9ba0945d23\ntime=2026-08-24T17:20:28.866Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2076032-mgmt-1/providers/Microsoft.Resources/deployments/3472b226dd1bf3f5c2c2fbd8f62937b0fe2f97cb4ca8a63168999f9ba0945d23/operationStatuses/08584140148567905734\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentNotFound\\\",\\n    \\\"message\\\": \\\"Deployment '3472b226dd1bf3f5c2c2fbd8f62937b0fe2f97cb4ca8a63168999f9ba0945d23' could not be found.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091954836298797056","occurred_at":"2026-08-24T18:15:46Z","signature_id":"0ec7efd378b135d7fa85cb93bcc2f610fe36f768df0ac4e2e0fe6122f4092f85","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928527292076032","occurred_at":"2026-08-24T16:32:12Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":4566,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.941747572815534,"anchor_occurrences":2,"after_last_push_count":15,"trend_counts":[2,1,7,2,1,0,2],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":46,"prior_last_seen_at":"2026-08-22T00:13:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-20 with version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00031f900\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tczrqg\" in resourcegroup=\"rg-np-version-upgrade-tczrqg-fhlnbg67qjlz\" to finish creating: GET https://rp.j7254272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d7b313-a7f0-40de-93b0-918f61dd2377\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d7b313-a7f0-40de-93b0-918f61dd2377\",\n      \"name\": \"c2d7b313-a7f0-40de-93b0-918f61dd2377\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T20:58:35.24134299Z\",\n      \"endTime\": \"2026-08-24T21:17:39.53487523Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-20 with version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00031f900\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tczrqg\" in resourcegroup=\"rg-np-version-upgrade-tczrqg-fhlnbg67qjlz\" to finish creating: GET https://rp.j7254272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d7b313-a7f0-40de-93b0-918f61dd2377\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d7b313-a7f0-40de-93b0-918f61dd2377\",\n      \"name\": \"c2d7b313-a7f0-40de-93b0-918f61dd2377\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T20:58:35.24134299Z\",\n      \"endTime\": \"2026-08-24T21:17:39.53487523Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:599]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e96960\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-q2tthc\" in resourcegroup=\"rg-np-downgrade-q2tthc-z7rbl7r2jwws\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47831462-3285-4a15-8722-8601b5708cf5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47831462-3285-4a15-8722-8601b5708cf5\",\n      \"name\": \"47831462-3285-4a15-8722-8601b5708cf5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:29:26.286804415Z\",\n      \"endTime\": \"2026-08-24T15:48:34.685207308Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:599]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e96960\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-q2tthc\" in resourcegroup=\"rg-np-downgrade-q2tthc-z7rbl7r2jwws\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47831462-3285-4a15-8722-8601b5708cf5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47831462-3285-4a15-8722-8601b5708cf5\",\n      \"name\": \"47831462-3285-4a15-8722-8601b5708cf5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:29:26.286804415Z\",\n      \"endTime\": \"2026-08-24T15:48:34.685207308Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977503907254272","occurred_at":"2026-08-24T19:54:38Z","signature_id":"eb508bccb119e029161a871dee87adcf17a97ac2d0acf65b89eac6aede0f896a","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"29b0688020a02097d8409be35d56bc8e657d156f249e7e0111a076d22256e797","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.941747572815534,"anchor_occurrences":2,"after_last_push_count":29,"trend_counts":[10,5,6,5,3,0,2],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":124,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T18:01:25Z\n  Ended:       2026-08-24T18:34:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T18:01:25Z\n  Ended:       2026-08-24T18:34:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T17:55:40Z\n  Ended:       2026-08-24T18:14:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0629888-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.209:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0629888/providers/microsoft.monitor/accounts/hcps-j0629888\", namespace=\"ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-tktc6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T17:55:40Z\n  Ended:       2026-08-24T18:15:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0629888-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.209:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0629888/providers/microsoft.monitor/accounts/hcps-j0629888\", namespace=\"ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-tktc6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"97ee0c31f908f1dbb20c9b6757ed42d4855cc07c6e4c69aed451babbcc2b480c","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"99e8b0c7fce35f618653d3b9144ebffd6644e1f77bf1ee7a7e41e85ca3b0c8eb","pr_number":6591,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.941747572815534,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,2],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-22T00:32:00Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T17:55:35Z\n  Ended:       2026-08-24T18:27:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6361088-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6361088-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T17:21:19Z\n  Ended:       2026-08-24T17:40:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9342336-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9342336-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932359686361088","occurred_at":"2026-08-24T16:54:34Z","signature_id":"a3d3dfe16c45487c7de4833dcff7c72d504e34a1b391992ceada6266956233ad","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091921367619342336","occurred_at":"2026-08-24T16:10:41Z","signature_id":"d4e52f6e7208e4a023c43c3ea9a95264341c256a3dcba5d703516c2b2c81f97b","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00133a200\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qp6jkgrm7jtk\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d814403-8a1a-482d-afa5-63e68ef4ed09\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d814403-8a1a-482d-afa5-63e68ef4ed09\",\n      \"name\": \"7d814403-8a1a-482d-afa5-63e68ef4ed09\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:13:13.654576687Z\",\n      \"endTime\": \"2026-08-24T21:32:17.358471293Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00133a200\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qp6jkgrm7jtk\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d814403-8a1a-482d-afa5-63e68ef4ed09\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d814403-8a1a-482d-afa5-63e68ef4ed09\",\n      \"name\": \"7d814403-8a1a-482d-afa5-63e68ef4ed09\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:13:13.654576687Z\",\n      \"endTime\": \"2026-08-24T21:32:17.358471293Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012a4ea0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-26mdm7dwkvm7\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fea58607-6b10-431a-a2a1-cc0c116e10b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fea58607-6b10-431a-a2a1-cc0c116e10b9\",\n      \"name\": \"fea58607-6b10-431a-a2a1-cc0c116e10b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:12:09.791388432Z\",\n      \"endTime\": \"2026-08-24T21:31:17.457093893Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012a4ea0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-26mdm7dwkvm7\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fea58607-6b10-431a-a2a1-cc0c116e10b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fea58607-6b10-431a-a2a1-cc0c116e10b9\",\n      \"name\": \"fea58607-6b10-431a-a2a1-cc0c116e10b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:12:09.791388432Z\",\n      \"endTime\": \"2026-08-24T21:31:17.457093893Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"ab3b51419f8536317834b49d26ead032c3d2df538686a40306860f09479fc7a4","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"fb242b6b09da99dc7298c2563fdd22c18c2c33f65971895aaf866ce3967b98d3","pr_number":6641,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"34e193d1e971458756c5fedbfac4e10a25751030aa42d3492b58c65f0ceda0fd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-10"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-14T20:39:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:304]: failed to create HCP cluster np-noedge-7pzvlw\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e28760\u003e: \n    failed to create HCP cluster np-noedge-7pzvlw: failed waiting for cluster=\"np-noedge-7pzvlw\" in resourcegroup=\"rg-np-noedge-7pzvlw-hmhh49x6v5k9\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n      \"name\": \"d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:29:01.361816726Z\",\n      \"endTime\": \"2026-08-24T21:48:06.132587418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:304]: failed to create HCP cluster np-noedge-7pzvlw\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e28760\u003e: \n    failed to create HCP cluster np-noedge-7pzvlw: failed waiting for cluster=\"np-noedge-7pzvlw\" in resourcegroup=\"rg-np-noedge-7pzvlw-hmhh49x6v5k9\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n      \"name\": \"d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:29:01.361816726Z\",\n      \"endTime\": \"2026-08-24T21:48:06.132587418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:95]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e95be0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-7w4qb7bx58wj\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c4e3b7e-7834-4bf7-9d07-9139f539790f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c4e3b7e-7834-4bf7-9d07-9139f539790f\",\n      \"name\": \"7c4e3b7e-7834-4bf7-9d07-9139f539790f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:28:50.851778461Z\",\n      \"endTime\": \"2026-08-24T21:47:56.22619205Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:95]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e95be0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-7w4qb7bx58wj\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c4e3b7e-7834-4bf7-9d07-9139f539790f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c4e3b7e-7834-4bf7-9d07-9139f539790f\",\n      \"name\": \"7c4e3b7e-7834-4bf7-9d07-9139f539790f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:28:50.851778461Z\",\n      \"endTime\": \"2026-08-24T21:47:56.22619205Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"7467fcd7d827463de8cbe0d7b5f551f437c25ba6ae7fc7de2730705826346049","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"ca98190d0c65ef7e152d8e004d3f56a8bcff3a1f40939b19e5d08cfc9a992c4b","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4a2f8bcf8d29c66434783789d75f6d18e6e94158f0e0f35e53c0353bce508345","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,2,0,0,0,0,2],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T18:19:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e70e20\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wmkndtkx988t\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\",\n      \"name\": \"a9d4baef-549e-4561-b748-439453d113af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:49:08.39871397Z\",\n      \"endTime\": \"2026-08-24T16:08:11.236941815Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e70e20\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wmkndtkx988t\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\",\n      \"name\": \"a9d4baef-549e-4561-b748-439453d113af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:49:08.39871397Z\",\n      \"endTime\": \"2026-08-24T16:08:11.236941815Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000408ec0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn9cdfvxcqll\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\",\n      \"name\": \"75726762-be88-402f-a564-60c91125dc8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:47:28.10624954Z\",\n      \"endTime\": \"2026-08-24T16:06:31.149377863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000408ec0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn9cdfvxcqll\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\",\n      \"name\": \"75726762-be88-402f-a564-60c91125dc8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:47:28.10624954Z\",\n      \"endTime\": \"2026-08-24T16:06:31.149377863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"7dd423d6ebe7dc246942b7ca078647aad56aaec8baea7f2d4aa8292a909f5f19","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"d07feba14b1358e1ba91e00ab3fcc20b89813cd03b2520364e81ecc050b70a83","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":23,"prior_last_seen_at":"2026-08-20T22:52:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6658,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b55233e1f233919ecc579e6967f76823775cbaf35c5ee37e0a8883b46726c324","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AllocationFailed; detail message The VM allocation failed due to an internal error. Please retry later or try deploying to a different location.; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-16T12:18:51Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-24T19:10:07.135Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T19:10:08.078Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T19:10:08.961Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6400512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05\ntime=2026-08-24T19:19:10.348Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05/operationStatuses/08584140082768889498\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/cluster-3vizzf2ahtjlc\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6400512-mgmt-2/agentPools/userswft1\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"AllocationFailed\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft1-38184404-vmss'. Please try alternative sizes 'Standard_E16ds_v6' (zones 2, 3, 1), 'Standard_E16ds_v6' (eastus, zones 1, 2, 3), 'Standard_E16ds_v6' (eastus2, zones 2, 1, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation.\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"AllocationFailed\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"3\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"The VM allocation failed due to an internal error. Please retry later or try deploying to a different location.\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091957267166400512","occurred_at":"2026-08-24T18:34:13Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bb424b290c173083428a34ac21b9472b0a9be4d1c592b771df2c71e46b269465","failure_pattern":"ERROR CODE: DeploymentFailed; detail code StorageAccountOperationInProgress; detail message An operation is currently performing on this storage account that requires exclusive access.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-24T16:09:49.176Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 description=\"Step infra\\n  Kind: ARM\\n  Template: templates/mgmt-infra.bicep\\n  Parameters: configurations/mgmt-infra.tmpl.bicepparam\"\ntime=2026-08-24T16:09:49.840Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2\ntime=2026-08-24T16:09:50.553Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 deployment=c783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1968512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290\ntime=2026-08-24T16:11:20.899Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/c783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290/operationStatuses/08584140190952500816\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcp-backups-storage\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcpBackupsStorageAccount\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcpBackupsStorageAccount\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"StorageAccountOperationInProgress\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"An operation is currently performing on this storage account that requires exclusive access.\\\\\\\"\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091914307531968512","occurred_at":"2026-08-24T15:43:31Z","signature_id":"d517e51fc5b09c819903bacd247416bcbbe167939922787f3a2438a125b3fdc2","pr_number":6659,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Timeout; detail message Operation is being Canceled due to timeout.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-08-24T07:13:06.981Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-08-24T07:13:09.018Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-08-24T07:13:10.536Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1860736%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59\ntime=2026-08-24T07:29:44.382Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1860736/providers/Microsoft.Resources/deployments/9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59/operationStatuses/08584140512958847265\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1860736/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1860736/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1860736\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091780518591860736","occurred_at":"2026-08-24T06:51:53Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6278,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36e7932077fe0de3697c04fb0ba302a7298b56f122f6c49db035b3d764e4980b","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentStackDeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nodepools (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nodepools (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-24T08:15:25.803Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nodepools stamp=2 description=\"Step nodepools\\n  Kind: ARMStack\\n  Template: templates/aks-nodepools.bicep\\n  Parameters: configurations/mgmt-cluster-nodepools.tmpl.bicepparam\"\ntime=2026-08-24T08:18:16.150Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nodepools stamp=2 err=\"stamp 2: failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\\",\\n  \\\"name\\\": \\\"67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: '676c3fe4-b0a1-44f5-8602-c3c05012b379'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2/providers/Microsoft.Resources/deployments/l4mv3scto7z6edigthkjdr62jbdrgoxoif33nvakxlwvg-260824084ipd0\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"OverconstrainedZonalAllocationRequest\\\",\\n            \\\"message\\\": \\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-41975975-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-41975975-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n  - Availability Zone\\\\n  - Differencing (Ephemeral) Disks\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n  - VM Size\\\\n\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091793135989428224","occurred_at":"2026-08-24T07:42:01Z","signature_id":"497ab79e6fe621da7f982b21f5dd28a45dc2f63297d3ed53abd84200879145f5","pr_number":6447,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c3db9c7d221b5f14f4cb052b432def48559d4c9a3619db1db26cc57a99142ed9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T04:39:49Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--wkl8qfr9l4cs/cluster-hs-pre-jtp7jx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c0960\u003e: \n    failed to create HCP cluster cluster-hs-pre-jtp7jx: failed waiting for cluster=\"cluster-hs-pre-jtp7jx\" in resourcegroup=\"rg-hs-pre--wkl8qfr9l4cs\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\",\n      \"name\": \"df253c01-fd54-4465-9725-47d0942d268e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:18:48.695053949Z\",\n      \"endTime\": \"2026-08-24T15:37:57.993163664Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--wkl8qfr9l4cs/cluster-hs-pre-jtp7jx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c0960\u003e: \n    failed to create HCP cluster cluster-hs-pre-jtp7jx: failed waiting for cluster=\"cluster-hs-pre-jtp7jx\" in resourcegroup=\"rg-hs-pre--wkl8qfr9l4cs\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\",\n      \"name\": \"df253c01-fd54-4465-9725-47d0942d268e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:18:48.695053949Z\",\n      \"endTime\": \"2026-08-24T15:37:57.993163664Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"113a28e0e41ddb16e2d1b2e27f386f97c1c16af7bfd0bd1391cd74394fd64a44","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-22T18:55:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a808e0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-cgt6b2gspgmg\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c64a33f9-1d34-4ca6-89af-ad50a67745b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c64a33f9-1d34-4ca6-89af-ad50a67745b6\",\n      \"name\": \"c64a33f9-1d34-4ca6-89af-ad50a67745b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:19:10.873312133Z\",\n      \"endTime\": \"2026-08-24T15:38:17.935941947Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-network-operator, redhat-operators-catalog, catalog-operator, openshift-route-controller-manager, ingress-operator, hosted-cluster-config-operator, packageserver, cluster-policy-controller, cluster-storage-operator, certified-operators-catalog, olm-operator, openshift-controller-manager, csi-snapshot-controller-operator, community-operators-catalog, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a808e0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-cgt6b2gspgmg\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c64a33f9-1d34-4ca6-89af-ad50a67745b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c64a33f9-1d34-4ca6-89af-ad50a67745b6\",\n      \"name\": \"c64a33f9-1d34-4ca6-89af-ad50a67745b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:19:10.873312133Z\",\n      \"endTime\": \"2026-08-24T15:38:17.935941947Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-network-operator, redhat-operators-catalog, catalog-operator, openshift-route-controller-manager, ingress-operator, hosted-cluster-config-operator, packageserver, cluster-policy-controller, cluster-storage-operator, certified-operators-catalog, olm-operator, openshift-controller-manager, csi-snapshot-controller-operator, community-operators-catalog, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"6b56df09ac5297e89b7853c7486e42c460930ae4b037271fddfef92b6ce0846a","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,1,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001580bc0\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-cbbn9kpl7fxv\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c6f112da-48e3-4cbb-a1b9-a47159e85ff5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c6f112da-48e3-4cbb-a1b9-a47159e85ff5\",\n      \"name\": \"c6f112da-48e3-4cbb-a1b9-a47159e85ff5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:18:01.504032647Z\",\n      \"endTime\": \"2026-08-24T15:37:08.017891025Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, azure-cloud-controller-manager, cluster-node-tuning-operator, olm-operator, olm-collect-profiles, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, community-operators-catalog, redhat-operators-catalog, machine-approver, openshift-controller-manager, cluster-storage-operator, dns-operator, redhat-marketplace-catalog, cluster-version-operator, openshift-route-controller-manager, ingress-operator, kube-scheduler, cluster-image-registry-operator, konnectivity-agent, catalog-operator, cluster-network-operator, ignition-server, ignition-server-proxy, openshift-apiserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001580bc0\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-cbbn9kpl7fxv\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c6f112da-48e3-4cbb-a1b9-a47159e85ff5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c6f112da-48e3-4cbb-a1b9-a47159e85ff5\",\n      \"name\": \"c6f112da-48e3-4cbb-a1b9-a47159e85ff5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:18:01.504032647Z\",\n      \"endTime\": \"2026-08-24T15:37:08.017891025Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, azure-cloud-controller-manager, cluster-node-tuning-operator, olm-operator, olm-collect-profiles, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, community-operators-catalog, redhat-operators-catalog, machine-approver, openshift-controller-manager, cluster-storage-operator, dns-operator, redhat-marketplace-catalog, cluster-version-operator, openshift-route-controller-manager, ingress-operator, kube-scheduler, cluster-image-registry-operator, konnectivity-agent, catalog-operator, cluster-network-operator, ignition-server, ignition-server-proxy, openshift-apiserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"14a501f15a848fe949827032cbe7eab1732753bec28bb5cacc0d3876ab277f28","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b652815635f16d983d1325dbd7e304560647aed7cf1be6254e8a2a57faf1d4fa","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef90e0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2zg88lxq9q4w\" to finish creating: GET https://rp.j3558656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\",\n      \"name\": \"b976f9f3-2891-4433-868a-76a8d23de317\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T16:54:08.369328068Z\",\n      \"endTime\": \"2026-08-24T17:13:08.208412685Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef90e0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2zg88lxq9q4w\" to finish creating: GET https://rp.j3558656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\",\n      \"name\": \"b976f9f3-2891-4433-868a-76a8d23de317\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T16:54:08.369328068Z\",\n      \"endTime\": \"2026-08-24T17:13:08.208412685Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091920407983558656","occurred_at":"2026-08-24T16:06:36Z","signature_id":"fdde03128e739a2f562ab5c3485dae63a0f561b3b4659197733a62ff1ea2a4e8","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4b645ca11afad5b79ec9dae25eb364818cdce5c3fe97c8a6de8581921ac37e5b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":90,"trend_counts":[26,14,33,40,14,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":84,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-nzmkwtbzb8xk/cluster-candidate-4-21-k47dlw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011d300\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-k47dlw: failed waiting for cluster=\"cluster-candidate-4-21-k47dlw\" in resourcegroup=\"rg-candidate-4-21-nzmkwtbzb8xk\" to finish creating: GET https://rp.j0629888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e224492a-2630-4a37-b5e0-7ef00f6cd9dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e224492a-2630-4a37-b5e0-7ef00f6cd9dd\",\n      \"name\": \"e224492a-2630-4a37-b5e0-7ef00f6cd9dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:34:09.560283017Z\",\n      \"endTime\": \"2026-08-24T17:53:13.775593008Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-nzmkwtbzb8xk/cluster-candidate-4-21-k47dlw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011d300\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-k47dlw: failed waiting for cluster=\"cluster-candidate-4-21-k47dlw\" in resourcegroup=\"rg-candidate-4-21-nzmkwtbzb8xk\" to finish creating: GET https://rp.j0629888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e224492a-2630-4a37-b5e0-7ef00f6cd9dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e224492a-2630-4a37-b5e0-7ef00f6cd9dd\",\n      \"name\": \"e224492a-2630-4a37-b5e0-7ef00f6cd9dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:34:09.560283017Z\",\n      \"endTime\": \"2026-08-24T17:53:13.775593008Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"9806f10f5f28de5b459cf7d30f0fa456f7ab916323bce3ad88103d4bb5b0c102","pr_number":6591,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-cbhfzqtvb9th/cluster-candidate-4-21-5bbfxw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea2680\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5bbfxw: failed waiting for cluster=\"cluster-candidate-4-21-5bbfxw\" in resourcegroup=\"rg-candidate-4-21-cbhfzqtvb9th\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n      \"name\": \"a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:28:32.844651935Z\",\n      \"endTime\": \"2026-08-24T17:47:34.560561952Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-cbhfzqtvb9th/cluster-candidate-4-21-5bbfxw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea2680\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5bbfxw: failed waiting for cluster=\"cluster-candidate-4-21-5bbfxw\" in resourcegroup=\"rg-candidate-4-21-cbhfzqtvb9th\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n      \"name\": \"a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:28:32.844651935Z\",\n      \"endTime\": \"2026-08-24T17:47:34.560561952Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"94e14a4948dabf46772c6e049c6c316329369f07279d585a70084b3a5b07d8b8","pr_number":6657,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bad17b00fd2861c199fb8c7332ad5eb58ff1eb35d64efc7066c892df3da78afa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] \u003cno_message\u003e; [clusterServiceStatus] \u003cno_message\u003e; [descendantResources] \u003cno_message\u003e; [hostedCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":7,"also_in":["STG"],"trend_counts":[1,1,2,2,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":19,"prior_last_seen_at":"2026-08-21T04:29:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c982a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-gdnhnw\" in resourcegroup=\"rg-hs-pre--hm7prkdxbkw6\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\",\n      \"name\": \"67ed1acc-0077-4b07-848d-484f46170f44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:21:50.42303027Z\",\n      \"endTime\": \"2026-08-24T15:45:59.872804145Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c982a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-gdnhnw\" in resourcegroup=\"rg-hs-pre--hm7prkdxbkw6\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\",\n      \"name\": \"67ed1acc-0077-4b07-848d-484f46170f44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:21:50.42303027Z\",\n      \"endTime\": \"2026-08-24T15:45:59.872804145Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"cf9ddb524cae2c0c6a37a93000fda2cd14fe56bff50daec6b5b3ed77ad21467c","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":215,"trend_counts":[43,30,18,17,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":168,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e8af48\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mb2sm98hvbm8\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n      \"name\": \"76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:14:43.826676557Z\",\n      \"endTime\": \"2026-08-24T15:38:50.051171369Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm25j05l3q2f8a1e83cjmn4lda8g still exists (deletion dispatched at 2026-08-24T15:14:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e8af48\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mb2sm98hvbm8\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n      \"name\": \"76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:14:43.826676557Z\",\n      \"endTime\": \"2026-08-24T15:38:50.051171369Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm25j05l3q2f8a1e83cjmn4lda8g still exists (deletion dispatched at 2026-08-24T15:14:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"53ff086f26d6ee86f70c417c33372b3bc3883205891aacb5f42319d457fb9cf5","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ba89a4166305f6c19c8eaadc8056c4375814afedeefcd4ac9c286060c7d5cad1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-10"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-14T20:23:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000ebc2a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-h4ldj6\" in resourcegroup=\"rg-candidate-5-0-xjnldggtrt7m\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\",\n      \"name\": \"fa0102ea-068c-4907-92f2-69579c623cac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:21:51.698922375Z\",\n      \"endTime\": \"2026-08-24T15:45:59.987785659Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm5ibpilvjmdnr77hbqf9t255bon still exists (deletion dispatched at 2026-08-24T15:21:51Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000ebc2a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-h4ldj6\" in resourcegroup=\"rg-candidate-5-0-xjnldggtrt7m\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\",\n      \"name\": \"fa0102ea-068c-4907-92f2-69579c623cac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:21:51.698922375Z\",\n      \"endTime\": \"2026-08-24T15:45:59.987785659Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm5ibpilvjmdnr77hbqf9t255bon still exists (deletion dispatched at 2026-08-24T15:21:51Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"60c407c2c17844ded754b0d5e8dda40f125391008bbbdd669c23fa4348b5b316","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[1,1,1,0,1,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-22T00:26:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010deff0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-4cmmkc\" in resourcegroup=\"admin-credential-lifecycle-test-mk5bt6ghv29k\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\",\n      \"name\": \"0644f278-df05-4603-a10d-3a66a1f851d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:03.672410989Z\",\n      \"endTime\": \"2026-08-24T15:41:10.046088761Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tjfmep0gog9fd4pfkr5bnpc0n642 still exists (deletion dispatched at 2026-08-24T15:17:04Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010deff0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-4cmmkc\" in resourcegroup=\"admin-credential-lifecycle-test-mk5bt6ghv29k\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\",\n      \"name\": \"0644f278-df05-4603-a10d-3a66a1f851d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:03.672410989Z\",\n      \"endTime\": \"2026-08-24T15:41:10.046088761Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tjfmep0gog9fd4pfkr5bnpc0n642 still exists (deletion dispatched at 2026-08-24T15:17:04Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"7df46629584e4c4253c91624b483f13865f970598c9ed547ff855ac9eea68fe2","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d3766fe34f03f22876c777e51a7be1734ee728533bfb3f28e111bf0b35fcad1b","failure_pattern":"UpdateHCPCluster20260630 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":60,"prior_last_seen_at":"2026-08-21T15:05:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00132c390\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00132c390\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"78869f45f7134e749985fc5b23dff71b456538a41d70734a716f1e608670ab6c","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":20,"prior_last_seen_at":"2026-08-22T14:52:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00396d0e0\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00396d0e0\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"dfb206f2ba6bc7bc3ce13c32e974065647bae3407bafdce87be1e26f94b6936e","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":48,"trend_counts":[16,6,8,5,3,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":190,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T18:15:21Z\n  Ended:       2026-08-24T18:36:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"21c20c66c7c9aa8660ddbc1f0e9827cf7f85ba5f2e371e49fb9660d2202ea0a2","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[2,2,2,0,2,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":35,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T18:02:20Z\n  Ended:       2026-08-24T18:38:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4cebd923-31c4-4c0a-b568-63bf1a281b9a\"\n  Description: Pod ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T18:03:20Z\n  Ended:       2026-08-24T18:38:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4cebd923-31c4-4c0a-b568-63bf1a281b9a\"\n  Description: Pod ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"9d5e8ec654bc2ebd399a25878bada85329c8658827f68d63b0042fb7498d8047","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[1,3,2,2,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":51,"prior_last_seen_at":"2026-08-21T05:44:12Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T14:40:10Z\n  Ended:       2026-08-24T14:49:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6438784-svc\", component=\"backend\", name=\"csstatedump\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879927736438784","occurred_at":"2026-08-24T13:26:52Z","signature_id":"f42ba6cef1b20012612134284fc91db6cdee01f2a97291f342457df1df0028e6","pr_number":6433,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","failure_pattern":"alert [svc] FleetControllerRetryHotLoop fired","search_query":"alert [svc] FleetControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[3,1,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-19T22:05:11Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T17:38:07Z\n  Ended:       2026-08-24T17:45:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j0629888-svc\", component=\"fleet\", name=\"clustersserviceregistrationcontroller\", severity=\"warning\"\n  Description: Fleet controller workqueue clustersserviceregistrationcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"3eebb3220fdafc37e59ae5d05cda26d2e052d0d1681fedb42f8fa69446913217","pr_number":6591,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[3,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-18T19:17:13Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-24T16:04:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-cxwt8 on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-24T16:04:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-jbtcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1ec91183-7c07-473b-b1d1-9df325a61c81\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-jbtcb on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-24T16:04:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-jbtcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1ec91183-7c07-473b-b1d1-9df325a61c81\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-jbtcb on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-24T16:04:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-cxwt8 on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"c239e70f5bf7ad0e42856cb2b9acc6d7c10a9ba7d6c9a4f5496cbd3609446018","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","failure_pattern":"alert [svc] KubeDaemonSetMisScheduled fired","search_query":"alert [svc] KubeDaemonSetMisScheduled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":65,"trend_counts":[15,9,9,10,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":207,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","occurrences":1}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:23:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-24T16:13:11Z\n  Ended:       2026-08-24T16:23:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"7caa0010541a7c4ff937ecc7f1700635aa06e8c5ef418d2f4d2c09799f427a78","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":69,"trend_counts":[15,12,10,11,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":228,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 24 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T16:11:08Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:25:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-08-24T16:13:07Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-08-24T16:13:07Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:26:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-08-24T16:17:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-08-24T16:17:09Z\n  Ended:       2026-08-24T16:23:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-08-24T16:17:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-08-24T16:17:09Z\n  Ended:       2026-08-24T16:29:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"367941eb104ee306ff1fe196c961bc8c4a584d69e0f31a0ee0696e981fbdf639","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[1,3,0,1,1,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":28,"prior_last_seen_at":"2026-08-22T04:44:09Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 6 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:21:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:22:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:22:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:23:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:22:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:23:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"a6a83541f08b4359551ade88cceb74431bde5d979f13cb6852b8c8bc1013fd98","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","failure_pattern":"alert [svc] KubeNodeUnreachable fired","search_query":"alert [svc] KubeNodeUnreachable fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":53,"trend_counts":[7,4,7,9,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":172,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T16:12:59Z\n  Ended:       2026-08-24T16:22:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"prometheus\", node=\"aks-userswft2-20578738-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft2-20578738-vmss000002 is unreachable and some workloads may be rescheduled.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T16:12:59Z\n  Ended:       2026-08-24T16:23:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"prometheus\", node=\"aks-userswft2-20578738-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft2-20578738-vmss000002 is unreachable and some workloads may be rescheduled."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"8ea4612bb5240c0f449b27c94dc170f61dc652ef24bc74880e6489a1d15ea2be","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-13T06:28:29Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T16:19:07Z\n  Ended:       2026-08-24T16:30:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n  Description: Pod arobit/arobit-forwarder-cxwt8 (fluentbit) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T16:20:06Z\n  Ended:       2026-08-24T16:33:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n  Description: Pod arobit/arobit-forwarder-cxwt8 (fluentbit) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"fe91f4f69f02671271e4cc6e4aaa4f966388d16d9e660f0bf8e6f1f0d5179d16","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fac505f52469073fa6a64250c64e44813d69018a355acebf1fdde81a505c53f0","failure_pattern":"expected 3 nodes to have original label key1=value1","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[2,1,0,2,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-21T15:05:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":15,"prior_last_seen_at":"2026-08-19T16:39:13Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-08-24T09:44:43.919Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T09:44:44.928Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-08-24T09:44:46.104Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=f40d7b7ab0b2d9646dc8b78fbaa252859442c0d4d4549c8765d66346f06ac123 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4585344-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff40d7b7ab0b2d9646dc8b78fbaa252859442c0d4d4549c8765d66346f06ac123\ntime=2026-08-24T10:14:43.424Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6655/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091810349014585344","occurred_at":"2026-08-24T08:50:25Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6655,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"911f609de0f6334698a9ae03ee21b22f86f10fb03ad2176871cac7bbc4b43a5f","failure_pattern":"failed to select node pool \u003cnodepool\u003e version for \u003cocp-channel\u003e channel: Get \"\u003curl\u003e\": dial tcp: lookup api","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-20T16:56:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.208.156:42093-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.208.156:42093-\u003e172.30.0.10:53: i/o timeout"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"7171e132341694fb10089af1503bd1696a386e558f7ad492e46e056500959d3b","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5d433008aba23c344781b86d474b8dbf122c3c1ce0be7c5095e73ef5b595140f","failure_pattern":"run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"\u003cvm\u003e\" (output: \"Client Version: v1.36.4 Kustomize Version: v5.8.1\")","search_query":"run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:160]: kubectl version should succeed from VM via private KAS internal LB (output: )\nUnexpected error:\n    \u003c*errors.errorString | 0xc0007fbf60\u003e: \n    run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:160]: kubectl version should succeed from VM via private KAS internal LB (output: )\nUnexpected error:\n    \u003c*errors.errorString | 0xc0007fbf60\u003e: \n    run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879954055696384","occurred_at":"2026-08-24T13:27:00Z","signature_id":"d079461036286dee8f18df87fbaabb5b38f1fcaa64aea1bbd3a2fe3d558a9ddb","pr_number":6429,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bff39f67862ef72eccdd3c2006d48756f417856635339107ace9fde9b0a679a4","failure_pattern":"stamp 2 scheduling must have requested resources","search_query":"stamp 2 scheduling must have requested resources","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have scheduling data for ready management clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.970873786407767,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have scheduling data for ready management clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:155 with:\nstamp 2 scheduling must have requested resources\nExpected\n    \u003cv1.ResourceList | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:155 with:\nstamp 2 scheduling must have requested resources\nExpected\n    \u003cv1.ResourceList | len:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"b261dd52f676523d8a989fb0761bdfb6871af15b6d8ed758e7d327e30440bdf8","pr_number":6657,"after_last_push_of_merged_pr":false}]}]},{"environment":"int","summary":{"total_runs":6,"failed_runs":2,"raw_occurrences":3,"matched_occurrences":3,"runs_affected":2},"rows":[{"environment":"int","failure_pattern_id":"13ca13618a2c4b8b7429b3001f7d02e419916c0961928bfe37af66198711655d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":7,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-03"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-07T23:58:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e43900\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fbxfthcgbs8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n      \"name\": \"7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T10:43:39.73808157Z\",\n      \"endTime\": \"2026-08-24T11:02:45.476245961Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e43900\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fbxfthcgbs8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n      \"name\": \"7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T10:43:39.73808157Z\",\n      \"endTime\": \"2026-08-24T11:02:45.476245961Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091834116805758976","occurred_at":"2026-08-24T10:24:52Z","signature_id":"3fa15812aed9d226c2b6bdead73b3a2c80d16497f78ee163ab8cbfa1074904ed","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"d414eaa781a1a73b840b0ceb0c494d2cfb36d6ce2f01987a892eec84026f3876","failure_pattern":"failed to select node pool \u003cnodepool\u003e version for \u003cocp-channel\u003e channel: Get \"\u003curl\u003e\": net/http: TLS handsh","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handsh","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handshake timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handshake timeout"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091834116805758976","occurred_at":"2026-08-24T10:24:52Z","signature_id":"c4114fce72966a1339b4e9230a4f088e9e9d8e5a8939f917174c276b7d0a2542","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":7,"also_in":["STG"],"trend_counts":[1,0,1,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-20T16:42:49Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-l8xph5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f20140\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-trf9f.apps.aro.r8g4l2d8f3g0v2i.b5du.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-l8xph5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f20140\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-trf9f.apps.aro.r8g4l2d8f3g0v2i.b5du.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091735435544891392","occurred_at":"2026-08-24T03:52:45Z","signature_id":"571e7c77690730a14fa42704d42877387a3fe5f21673f5f238b11729008a3c86","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":12,"matched_occurrences":12,"runs_affected":2},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-gm6hwh, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: failed waiting for cluster=\"cluster-candidate-5-0-gm6hwh\" in resourcegroup=\"rg-candidate-5-0-mqrq7j46wgxv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":32,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001f06450\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-m4qs88j7zlw8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001f06450\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-m4qs88j7zlw8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-c58j4775b54x/cluster-nightly-5-1-rb2jwr should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a0d800\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-rb2jwr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: failed waiting for cluster=\"cluster-nightly-5-1-rb2jwr\" in resourcegroup=\"rg-nightly-5-1-c58j4775b54x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-c58j4775b54x/cluster-nightly-5-1-rb2jwr should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a0d800\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-rb2jwr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: failed waiting for cluster=\"cluster-nightly-5-1-rb2jwr\" in resourcegroup=\"rg-nightly-5-1-c58j4775b54x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-mqrq7j46wgxv/cluster-candidate-5-0-gm6hwh should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001400690\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-gm6hwh, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: failed waiting for cluster=\"cluster-candidate-5-0-gm6hwh\" in resourcegroup=\"rg-candidate-5-0-mqrq7j46wgxv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-mqrq7j46wgxv/cluster-candidate-5-0-gm6hwh should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001400690\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-gm6hwh, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: failed waiting for cluster=\"cluster-candidate-5-0-gm6hwh\" in resourcegroup=\"rg-candidate-5-0-mqrq7j46wgxv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"48cf5ec4badd8d65b424a299cc4f30c373af95f5d42c09e6725e0ee15b8c1667","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"a75e2b44cb494e9ca286b15ddee1a744c32c76fd89cb2c425050bd89a5961c7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"5fc7819289b1249d9f2299f61167ea9b6a16c06d2102e06da9f3562ee86b0ad0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f966d8a6f1dc892e4c6ee21571e53ba7d8d1ed39b375430e469f3616f54b9b95","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-7z565t\" to 4.21.0-0.nightly-multi-2026-08-24-143120\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b5de0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-7z565t\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-98477z6czjf7\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n      \"name\": \"d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:45:23.595556181Z\",\n      \"endTime\": \"2026-08-24T23:45:30.682388649Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-7z565t\" to 4.21.0-0.nightly-multi-2026-08-24-143120\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b5de0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-7z565t\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-98477z6czjf7\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n      \"name\": \"d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:45:23.595556181Z\",\n      \"endTime\": \"2026-08-24T23:45:30.682388649Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"15bac440b3d4b1dd68415a557e13f0d806f6cdb816f9edcd9423e02924b01f2e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d8c04d75e2d056f1b8b36668a13ab7045b056c9f740c7013c4882cfcfef5d2d6","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-cbj2dm\" to 4.22.0-0.nightly-multi-2026-08-23-080550\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000df2720\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-cbj2dm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4ls4ksxqchgg\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n      \"name\": \"d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:48:14.416516082Z\",\n      \"endTime\": \"2026-08-24T23:48:20.787265431Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-cbj2dm\" to 4.22.0-0.nightly-multi-2026-08-23-080550\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000df2720\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-cbj2dm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4ls4ksxqchgg\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n      \"name\": \"d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:48:14.416516082Z\",\n      \"endTime\": \"2026-08-24T23:48:20.787265431Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"498a13bc157b497dedc0c407f7c032fb09b28d4110c6fbee95fdae33ff5fe349","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"724e7a51f2cd05d0f0c351e24f28344ba4eb40490e6de7ed02ace0e7a7856472","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-lrdz8v\" to 4.23.0-0.nightly-multi-2026-08-24-110923\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000881340\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-lrdz8v\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-t25q8kvqvdhw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\",\n      \"name\": \"4393d587-2d85-4674-a4f6-155ceb1138f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:48:28.850341951Z\",\n      \"endTime\": \"2026-08-24T23:48:30.944709051Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-lrdz8v\" to 4.23.0-0.nightly-multi-2026-08-24-110923\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000881340\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-lrdz8v\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-t25q8kvqvdhw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\",\n      \"name\": \"4393d587-2d85-4674-a4f6-155ceb1138f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:48:28.850341951Z\",\n      \"endTime\": \"2026-08-24T23:48:30.944709051Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"6663bc04df6e96cb243dd9f35fd825b118edab670cd360f1ba5fc79ade7a0f44","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3b509588f678a0fd976eb453004a3e8a55c0be5c12f8340c463453a657ce6678","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-q6hwb9\" to 5.1.0-0.nightly-multi-2026-08-24-111752\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a25da0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-q6hwb9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-kgxwzfcjcc2b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n      \"name\": \"7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:51:24.1449981Z\",\n      \"endTime\": \"2026-08-24T23:51:30.86207539Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-q6hwb9\" to 5.1.0-0.nightly-multi-2026-08-24-111752\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a25da0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-q6hwb9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-kgxwzfcjcc2b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n      \"name\": \"7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:51:24.1449981Z\",\n      \"endTime\": \"2026-08-24T23:51:30.86207539Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"05526aa86d5e4dd680e08fb3cf223532281a30bb281575b5c8abfec129525d60","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"beaf47bd37dc21f47808a0240288ea4322da110d92ca67bf08de0f6793353753","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.35; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,1,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-96fcvx\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008e5080\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-qkzx8x2fwwlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-96fcvx\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-96fcvx\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008e5080\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-qkzx8x2fwwlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-96fcvx\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"67be8c6caed95a193a1745532d025891093937507a496524f899a7d2c80f89f2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5443585aedc88f5d2589a8fb3795e77266206bcfdb680d3cfbedccaf7bc67f50","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.30; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,1,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-znvqfb\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008e8630\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-bb24w6nmgn9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-znvqfb\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-znvqfb\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008e8630\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-bb24w6nmgn9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-znvqfb\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"eabd42ab0d34b7f1d8ca082eb227c6c5a44990af4cb2ad49ebe668cc8663e6ed","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"2b6de8e3eeb7b1eb9cb9733a09ec9f907952649e2f47b910e6bb29761a829864","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.10; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,1,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-gmlx25\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000703a40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b6dm7hck8ksd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-gmlx25\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-gmlx25\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000703a40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b6dm7hck8ksd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-gmlx25\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"233817672ec9f15c61036dd69be2232e5da454215c4ae03262d5d9d1147fe964","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6c0df54cac07833305ae54d7a30bcf9a55390b618040466bd5f2843d80e698a8","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,1,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000ad6720\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000ad6720\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"3fd31e6b902a75b339639f3cb28f3e152b295a16939ec2f7056ea2da7a8f4e73","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6411b2b281a97b7e65400b269143638faa6befdec2dd319ac1327622333be049","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":5,"also_in":["STG"],"trend_counts":[0,0,1,1,1,1,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002b4c960\u003e: \n    node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1.35.4\"); o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002b4c960\u003e: \n    node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1.35.4\"); o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"0c1e40a7688ce333e1802f99689a7431268dce71db699919518e6ab02caaba1e","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":3,"failed_runs":2,"raw_occurrences":19,"matched_occurrences":19,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":8,"after_last_push_count":55,"also_in":["INT"],"trend_counts":[0,0,2,10,17,18,8],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001990018\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-58vq7.apps.aro.cilium-cluster.xjbu.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001990018\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-58vq7.apps.aro.cilium-cluster.xjbu.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0023089a8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gd2m6.apps.aro.agg-cluster.sh9x.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0023089a8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gd2m6.apps.aro.agg-cluster.sh9x.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002bc60a8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2xkbq.apps.aro.cilium-cl.yqg3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002bc60a8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2xkbq.apps.aro.cilium-cl.yqg3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"128bdc5bd9fd4d676037c7b3876358423100506abaac1cb7c6a2aa6dca99f9d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"268af4cfefc9d9ce5477c6309e34d9e625934c5fec5612737f144e143459ca23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"568397239d373e84cc1193077022f2929fdbc7536d41e0a023cda085a91923ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"64d7370897c1ddb0a4c082a98b5de0abd894ee5bd836383e651a1f53f91a033d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"6f400eea163c9e3d28c8b1f64accc1cf5963abf78c260413f5fe388a60f0006c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"aa4dc686c4f2ffb192f550321a5d618c671a84350ba22f26b4ee1aaac5686fb0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"c217516f086c4ee518f66b5fc0b63d7e19723573004fb051ee9ac9e2e83e0fb9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"de74f8de034a6e8da10a240f9e0d6f03ee3be2522666aaf0b6ad709ddf3efbc4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":4,"after_last_push_count":19,"trend_counts":[0,3,0,4,4,4,4],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0012c61b0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-23-080550, targetMinor=4.23.0-0.nightly-multi-2026-08-24-110923) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-24-110923\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0012c61b0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-23-080550, targetMinor=4.23.0-0.nightly-multi-2026-08-24-110923) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-24-110923\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000013c68\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-24-135049, targetMinor=4.21.0-0.nightly-multi-2026-08-24-143120) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-24-143120\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000013c68\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-24-135049, targetMinor=4.21.0-0.nightly-multi-2026-08-24-143120) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-24-143120\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001423c38\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-24-143120, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001423c38\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-24-143120, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"1f961d63e5efbc9f674297ab6ff360e18191616cad68297a32bfc0352b1efd45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"5d51d6400541f1ef2ea735bc34f88c2cca7f1b91d8f8680efed7dfe0ab6e075d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"8c228d79217e5703c43c091d1bbcb6c74c0d2e48dd70a8ed3bbc113b8600d236","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"9a6af90baac0bf0367bb530e542307559248b48af09dc4fa5550242ec09f64e4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":146,"also_in":["DEV"],"trend_counts":[0,0,4,1,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":13,"prior_last_seen_at":"2026-08-21T23:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8ea40\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf6jg45wtwxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\",\n      \"name\": \"62db7451-86ec-4c6a-9227-bc775f891ebb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:05:01.131009318Z\",\n      \"endTime\": \"2026-08-24T23:24:05.637766062Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8ea40\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf6jg45wtwxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\",\n      \"name\": \"62db7451-86ec-4c6a-9227-bc775f891ebb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:05:01.131009318Z\",\n      \"endTime\": \"2026-08-24T23:24:05.637766062Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"ce363ec1f1315846ac4ca18a663f8ca57451d8f5854815877a996fbdf3f45dd9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"db86d0a7b49ef93daf7cdb63d12e83b79ddcd71210e2847a436b8d80dd5dde7f","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] \u003cno_message\u003e; [clusterServiceStatus] \u003cno_message\u003e; [descendantResources] \u003cno_message\u003e; [hostedCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001716270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nml9dcvzzfr9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n      \"name\": \"d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:14:11.789973414Z\",\n      \"endTime\": \"2026-08-24T23:38:15.678890881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001716270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nml9dcvzzfr9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n      \"name\": \"d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:14:11.789973414Z\",\n      \"endTime\": \"2026-08-24T23:38:15.678890881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"3feed0e49b703d23a4823f602dec7fe62c46bf78411b06f90b5ee2c2bde72da5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,3,2,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-20T08:35:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00118e480\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-h759ch\" in resourcegroup=\"rg-nightly-4-20-mzr4r5lz42qp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\",\n      \"name\": \"55ed743f-d106-4adb-b590-caa3369caeb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:25:48.884097042Z\",\n      \"endTime\": \"2026-08-24T23:49:50.709200875Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd7vm7aq4os1ksjghr3o1msj3ksno6l still exists (deletion dispatched at 2026-08-24T23:25:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00118e480\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-h759ch\" in resourcegroup=\"rg-nightly-4-20-mzr4r5lz42qp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\",\n      \"name\": \"55ed743f-d106-4adb-b590-caa3369caeb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:25:48.884097042Z\",\n      \"endTime\": \"2026-08-24T23:49:50.709200875Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd7vm7aq4os1ksjghr3o1msj3ksno6l still exists (deletion dispatched at 2026-08-24T23:25:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"6a0460b71f8ba99286143da10d9b371325b39609e9612fa043b5980a861407e3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b6258cd488b080bf5ba7099ce8387b9d2f4d222d926f9a0c81022dc736e89db2","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,0,0,0,2,2,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017be510\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017be510\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"c14f9a5639d319819baa7a308cbb789519348e32b0abac6b02f5611b9187fbe7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c75b40567502299dd397c0248cce6097e2377df763d3deec7ae8c7f9cd09e3b2","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0012c0d10\u003e: \n    node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]},{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0012c0d10\u003e: \n    node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]},{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"5565031e089c6eb68fe43eb98d2d43776a82305498e8c6a0a7f4c918b0348bf0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":26,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002ac4480\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002ac4480\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"1ecfdcaa8fd2583c7f65498931069ae0e42fa1b933a7ed7714a974ea82a7b373","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,0,1,1,2,2,1],"trend_range":"2026-08-18..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}