{"meta":{"date":"2026-09-25","timezone":"UTC","generated_at":"2026-10-14T18:26:54Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":149,"failed_runs":121,"runs_with_occurrences":121,"runs_with_failure_pattern_matches":108,"runs_with_unmatched_occurrences":13,"runs_without_occurrence_rows":28,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103632247545401344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7185,"pr_state":"closed","pr_sha":"f50d244f4eedc226cd00ae3148a09ec7298eb255","final_merged_sha":"82a0c5b832cb3e6259e0c6c1090e6105a0f71753","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T23:46:26Z","started_at":"2026-09-25T23:46:26Z","completed_at":"2026-09-26T00:46:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T00:55:00.505075912Z","timing_metadata_checked_at":"2026-09-26T00:55:00.505075912Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T00:55:00.507646104Z","region_metadata_checked_at":"2026-09-26T00:55:00.507646104Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c2f0545eba730bd98c0412e33652420f49148916eb1c817f4b320ccb3fc52fda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103632247545401344","occurred_at":"2026-09-25T23:46:26Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-26T00:07:26.244Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T00:07:27.744Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T00:07:28.475Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c6f6a0e9ed66576db75e4f5edd9325023ba1bd00cc1573f55cc43b013f3d8254 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5401344%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc6f6a0e9ed66576db75e4f5edd9325023ba1bd00cc1573f55cc43b013f3d8254\ntime=2026-09-26T00:10:59.556Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5401344/providers/Microsoft.Resources/deployments/c6f6a0e9ed66576db75e4f5edd9325023ba1bd00cc1573f55cc43b013f3d8254/operationStatuses/08584112256374751743\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5401344/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5401344/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5401344\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report c3d253d0-b4f4-4213-be26-e137eaf24c7f:9/26/2026 12:08:41 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T00:12:59.562Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T00:13:00.295Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T00:13:01.023Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2c4f9afc32064e7fbde0098252476ca4f2c555e802c124a268312269ae74ced5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5401344%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2c4f9afc32064e7fbde0098252476ca4f2c555e802c124a268312269ae74ced5\ntime=2026-09-26T00:29:35.447Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5401344/providers/Microsoft.Resources/deployments/2c4f9afc32064e7fbde0098252476ca4f2c555e802c124a268312269ae74ced5/operationStatuses/08584112253049438015\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5401344/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5401344/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5401344\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 6d273900-3e80-430e-9e9b-cc980a819241:9/26/2026 12:28:54 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T00:31:35.447Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T00:31:46.002Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T00:31:47.038Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=b9caeeac5f96c466c7656ddeb7452f11f9d047bb47865f52e598d619e4e0310e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5401344%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb9caeeac5f96c466c7656ddeb7452f11f9d047bb47865f52e598d619e4e0310e\ntime=2026-09-26T00:37:25.849Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103630683309084672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"19695b04c65ccb146884b20473c81d0f7085ad28","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T23:40:13Z","started_at":"2026-09-25T23:40:13Z","completed_at":"2026-09-26T00:44:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T00:55:00.293439173Z","timing_metadata_checked_at":"2026-09-26T00:55:00.293439173Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T00:55:00.296493163Z","region_metadata_checked_at":"2026-09-26T00:55:00.296493163Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0ec2974340539e3d0b76e498863c49c507b5d42c668b5352bcb4341c0f0eeb98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103630683309084672","occurred_at":"2026-09-25T23:40:13Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-26T00:05:44.311Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T00:05:46.720Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T00:05:48.319Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f4b607d0d02c46670f1d1fdf0696848d6417d4c2382cde6526a8d79614cac1a7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9084672%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff4b607d0d02c46670f1d1fdf0696848d6417d4c2382cde6526a8d79614cac1a7\ntime=2026-09-26T00:08:19.373Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9084672/providers/Microsoft.Resources/deployments/f4b607d0d02c46670f1d1fdf0696848d6417d4c2382cde6526a8d79614cac1a7/operationStatuses/08584112257382371931\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9084672/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9084672/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9084672\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 0fdc339d-1c4e-4de3-b77d-f47ecab3fef7:9/26/2026 12:07:04 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T00:10:19.375Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T00:10:20.118Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T00:10:21.694Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=3269581d185d03931c9b79cc66339132c519a978e6f6cda46cb80eebc3d7837d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9084672%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3269581d185d03931c9b79cc66339132c519a978e6f6cda46cb80eebc3d7837d\ntime=2026-09-26T00:25:57.330Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9084672/providers/Microsoft.Resources/deployments/3269581d185d03931c9b79cc66339132c519a978e6f6cda46cb80eebc3d7837d/operationStatuses/08584112254648539999\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9084672/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9084672/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9084672\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 46b4a858-284e-4e1b-b169-e197dc0eb47c:9/26/2026 12:25:39 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T00:27:57.330Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T00:27:58.460Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T00:27:59.909Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f623b6f27eafd69dba9c767544d134313b90ae203320d5f11a93ac200693032d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9084672%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff623b6f27eafd69dba9c767544d134313b90ae203320d5f11a93ac200693032d\ntime=2026-09-26T00:35:43.999Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103630333239889920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7187,"pr_state":"open","pr_sha":"bf7f46462c66b992eb0ed0e4ace7ca43fcd6d8a6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T23:38:49Z","started_at":"2026-09-25T23:38:49Z","completed_at":"2026-09-26T00:47:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T00:55:00.702467896Z","timing_metadata_checked_at":"2026-09-26T00:55:00.702467896Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T00:55:00.705146088Z","region_metadata_checked_at":"2026-09-26T00:55:00.705146088Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8e515e42d963cbe7b9ee5ed8a377a2dd362d22191e74ac2af792bad462b79cb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103630333239889920","occurred_at":"2026-09-25T23:38:49Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-26T00:11:52.987Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T00:11:54.672Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T00:11:55.459Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8e7952d540cbe777236c9644310d488aefae2fd308040a879252afa860e9ec88 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9889920%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8e7952d540cbe777236c9644310d488aefae2fd308040a879252afa860e9ec88\ntime=2026-09-26T00:14:56.175Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9889920/providers/Microsoft.Resources/deployments/8e7952d540cbe777236c9644310d488aefae2fd308040a879252afa860e9ec88/operationStatuses/08584112253704864966\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9889920/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9889920/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9889920\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 9ff84196-1c4e-41dd-a6f2-0a17e70f2462:9/26/2026 12:12:58 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T00:16:56.181Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T00:16:56.918Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T00:16:58.061Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8129d6c1c6edbc3c4d6f196bbe14c57681a45bafc6a3165af424b44cfbdb204a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9889920%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8129d6c1c6edbc3c4d6f196bbe14c57681a45bafc6a3165af424b44cfbdb204a\ntime=2026-09-26T00:32:31.996Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9889920/providers/Microsoft.Resources/deployments/8129d6c1c6edbc3c4d6f196bbe14c57681a45bafc6a3165af424b44cfbdb204a/operationStatuses/08584112250680737189\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9889920/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9889920/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9889920\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report c6cc3291-7c3b-48c1-a2de-ec48a0327d16:9/26/2026 12:32:03 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T00:34:31.997Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T00:34:33.135Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T00:34:33.952Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=1e08d522a757546e7297fa1b2c3574a69227d0f73489fac16a6eebd7e354f5a5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9889920%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F1e08d522a757546e7297fa1b2c3574a69227d0f73489fac16a6eebd7e354f5a5\ntime=2026-09-26T00:41:52.735Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103629622468939776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7188,"pr_state":"closed","pr_sha":"1a692710f89c9c1ffcd99d3b48a7b09acd258143","final_merged_sha":"1a692710f89c9c1ffcd99d3b48a7b09acd258143","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T23:36:00Z","started_at":"2026-09-25T23:36:00Z","completed_at":"2026-09-26T00:30:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T00:44:30.215492598Z","timing_metadata_checked_at":"2026-09-26T00:44:30.215492598Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T00:44:30.218595889Z","region_metadata_checked_at":"2026-09-26T00:44:30.218595889Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ae0858517d594ae71f7f231a69f77134fc163361ebcd2c3651a59503df047d24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103629622468939776","occurred_at":"2026-09-25T23:36:00Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T23:55:18.457Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:55:19.896Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:55:20.643Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=6afb3655b68312bc2e39f973637d372bf476b3c736390b3287be4f9274ae09f4 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8939776%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6afb3655b68312bc2e39f973637d372bf476b3c736390b3287be4f9274ae09f4\ntime=2026-09-25T23:58:51.475Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8939776/providers/Microsoft.Resources/deployments/6afb3655b68312bc2e39f973637d372bf476b3c736390b3287be4f9274ae09f4/operationStatuses/08584112263653208424\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8939776/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8939776/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8939776\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 06a1476a-b5ff-4ebe-9d45-2bd9b793ae84:9/25/2026 11:56:27 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T00:00:51.479Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T00:00:52.420Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T00:00:53.572Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9795985625934e10e57927582931cc231d6c4af0542581f1167ada0dc9d31702 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8939776%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9795985625934e10e57927582931cc231d6c4af0542581f1167ada0dc9d31702\ntime=2026-09-26T00:17:28.329Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8939776/providers/Microsoft.Resources/deployments/9795985625934e10e57927582931cc231d6c4af0542581f1167ada0dc9d31702/operationStatuses/08584112260326248473\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8939776/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8939776/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8939776\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 82827fa9-10aa-4cae-86af-fc965736094a:9/26/2026 12:16:47 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T00:19:28.333Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T00:19:29.519Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T00:19:30.564Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=af4276016b326e7cb6c6dad8ac4189a9f4f08946c3c89526700655a94146ab69 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8939776%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Faf4276016b326e7cb6c6dad8ac4189a9f4f08946c3c89526700655a94146ab69\ntime=2026-09-26T00:25:18.135Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103622374401249280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"1b17ed628e30d424ef32118c72f436f418bcf921","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T23:07:12Z","started_at":"2026-09-25T23:07:12Z","completed_at":"2026-09-26T00:04:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T00:23:06.196884814Z","timing_metadata_checked_at":"2026-09-26T00:23:06.196884814Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T00:23:06.200641104Z","region_metadata_checked_at":"2026-09-26T00:23:06.200641104Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"64d55f611ec0deb6b91565d5112b1bc151c71a24839792f7dc57b80b9e970b57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103622374401249280","occurred_at":"2026-09-25T23:07:12Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T23:29:03.951Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:29:05.712Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:29:06.479Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=a2bf3c0e8c0215e6b72da9a2c497392c100b598bbae4c4a88e8aee332372dbe1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1249280%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fa2bf3c0e8c0215e6b72da9a2c497392c100b598bbae4c4a88e8aee332372dbe1\ntime=2026-09-25T23:32:37.291Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1249280/providers/Microsoft.Resources/deployments/a2bf3c0e8c0215e6b72da9a2c497392c100b598bbae4c4a88e8aee332372dbe1/operationStatuses/08584112279395043420\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1249280/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1249280/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1249280\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report b35c2fbf-796c-4f91-8c25-37e8c6b71a8b:9/25/2026 11:30:16 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T23:34:37.295Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:34:38.039Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:34:38.861Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f86cd5f536e01bebd1744b84507e328b29840d84acf18892289493bd6bd87ce4 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1249280%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff86cd5f536e01bebd1744b84507e328b29840d84acf18892289493bd6bd87ce4\ntime=2026-09-25T23:50:41.825Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1249280/providers/Microsoft.Resources/deployments/f86cd5f536e01bebd1744b84507e328b29840d84acf18892289493bd6bd87ce4/operationStatuses/08584112276071368168\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1249280/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1249280/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1249280\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 62ac70ed-32cd-42c2-b357-02d2af5debf2:9/25/2026 11:50:28 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T23:52:41.828Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:52:42.711Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:52:43.750Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=562deaf7f4b0dcfad8d7836c1d10989460f99d582c1a397a8555cec74a24b2f3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1249280%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F562deaf7f4b0dcfad8d7836c1d10989460f99d582c1a397a8555cec74a24b2f3\ntime=2026-09-25T23:59:03.748Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103618033778954240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"1f52bad80d3779b1af0809af78b4a5b351d87b4c","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T22:49:57Z","started_at":"2026-09-25T22:49:57Z","completed_at":"2026-09-25T23:50:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T00:01:32.728943244Z","timing_metadata_checked_at":"2026-09-26T00:01:32.728943244Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T00:01:32.732063134Z","region_metadata_checked_at":"2026-09-26T00:01:32.732063134Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d1a163328e0011654e6882b0dabc05c4ed2d2040e34b4f7b28076f347856bd7b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103618033778954240","occurred_at":"2026-09-25T22:49:57Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T23:15:04.023Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:15:06.056Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:15:07.434Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=67697682d2439cd7994c70d2ade468ba6d9d2021831f978250d3962a6181b240 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8954240%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F67697682d2439cd7994c70d2ade468ba6d9d2021831f978250d3962a6181b240\ntime=2026-09-25T23:17:38.706Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8954240/providers/Microsoft.Resources/deployments/67697682d2439cd7994c70d2ade468ba6d9d2021831f978250d3962a6181b240/operationStatuses/08584112287789256016\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8954240/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8954240/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8954240\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report a3275ef1-cc6a-4e98-bc33-acddeaaee42c:9/25/2026 11:16:15 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T23:19:38.713Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:19:39.984Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:19:41.346Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=49433b0e3b6a43b92533a2bb7e00295cede773d22ba54032114fb47770ed4814 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8954240%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F49433b0e3b6a43b92533a2bb7e00295cede773d22ba54032114fb47770ed4814\ntime=2026-09-25T23:35:16.317Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8954240/providers/Microsoft.Resources/deployments/49433b0e3b6a43b92533a2bb7e00295cede773d22ba54032114fb47770ed4814/operationStatuses/08584112285049959840\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8954240/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8954240/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8954240\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 680ac4ad-1652-434a-894b-26ebd57d3fab:9/25/2026 11:34:51 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T23:37:16.317Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:37:17.270Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:37:19.005Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=b00fff88d038c33c5af9214bfc8cb1dfa66b7877a47f266d75c9a1340d606c51 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8954240%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb00fff88d038c33c5af9214bfc8cb1dfa66b7877a47f266d75c9a1340d606c51\ntime=2026-09-25T23:45:03.723Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103618006578892800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7174,"pr_state":"closed","pr_sha":"756dbd0aea05ea7d18d18b94d48e2ccdb7df1c9c","final_merged_sha":"756dbd0aea05ea7d18d18b94d48e2ccdb7df1c9c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T22:49:50Z","started_at":"2026-09-25T22:49:50Z","completed_at":"2026-09-25T23:35:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T23:51:12.168986744Z","timing_metadata_checked_at":"2026-09-25T23:51:12.168986744Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T23:51:12.172173834Z","region_metadata_checked_at":"2026-09-25T23:51:12.172173834Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5161f41cffdc624f5f4559874b626ad561cda937abca58e35bfa029c6da327e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103618006578892800","occurred_at":"2026-09-25T22:49:50Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T23:00:43.901Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:00:45.309Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:00:46.147Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=a593dc7f3fd319268bca9dfdc2ae92834da559a174db184ef5833a6c398786cd portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8892800%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fa593dc7f3fd319268bca9dfdc2ae92834da559a174db184ef5833a6c398786cd\ntime=2026-09-25T23:04:16.922Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8892800/providers/Microsoft.Resources/deployments/a593dc7f3fd319268bca9dfdc2ae92834da559a174db184ef5833a6c398786cd/operationStatuses/08584112296398044830\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8892800/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8892800/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8892800\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 93677b9c-e6b3-442b-9111-dcae8ae62626:9/25/2026 11:01:45 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T23:06:16.925Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:06:17.750Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:06:18.502Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=1174b8e05a1d0caf34d583be842c0d060bcab0abc1b022e9109bf466c0578f8e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8892800%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F1174b8e05a1d0caf34d583be842c0d060bcab0abc1b022e9109bf466c0578f8e\ntime=2026-09-25T23:22:22.108Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8892800/providers/Microsoft.Resources/deployments/1174b8e05a1d0caf34d583be842c0d060bcab0abc1b022e9109bf466c0578f8e/operationStatuses/08584112293074633710\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8892800/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8892800/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8892800\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report b25111c8-815c-4e53-ae50-032feedc9ce9:9/25/2026 11:22:07 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T23:24:22.117Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:24:22.692Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:24:23.675Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d5668fd14d4d309bc9ab0d966c8bb46b1c2f065fae88a14e0942b1790f35ac9c portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8892800%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd5668fd14d4d309bc9ab0d966c8bb46b1c2f065fae88a14e0942b1790f35ac9c\ntime=2026-09-25T23:30:43.699Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7182/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103612428464951296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7182,"pr_state":"closed","pr_sha":"b56a252408226d8ed3ec180421d2845783ace28c","final_merged_sha":"b56a252408226d8ed3ec180421d2845783ace28c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T22:27:40Z","started_at":"2026-09-25T22:27:40Z","completed_at":"2026-09-25T23:25:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T23:41:04.216083002Z","timing_metadata_checked_at":"2026-09-25T23:41:04.216083002Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T23:41:04.21977389Z","region_metadata_checked_at":"2026-09-25T23:41:04.21977389Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c84ee1b5b36ce8d17f296303e9df43d76b452e004272d66c7118bc4f1195229f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7182/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103612428464951296","occurred_at":"2026-09-25T22:27:40Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T22:49:21.532Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:49:23.124Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:49:24.571Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=58547af0e40efb140e106c4a983532d62eae0bda579fe35e8041a8d7b7beb7c3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4951296%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F58547af0e40efb140e106c4a983532d62eae0bda579fe35e8041a8d7b7beb7c3\ntime=2026-09-25T22:52:56.258Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4951296/providers/Microsoft.Resources/deployments/58547af0e40efb140e106c4a983532d62eae0bda579fe35e8041a8d7b7beb7c3/operationStatuses/08584112303218419009\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4951296/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4951296/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j4951296\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 649af4c0-f7ab-4027-ad38-48d765f10467:9/25/2026 10:50:32 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:54:56.264Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:54:57.304Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:54:58.687Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d126cc0672b0147496e6a4c360a12354eae7e77c1c3cce43ee0cbd8363b057ff portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4951296%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd126cc0672b0147496e6a4c360a12354eae7e77c1c3cce43ee0cbd8363b057ff\ntime=2026-09-25T23:10:33.411Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4951296/providers/Microsoft.Resources/deployments/d126cc0672b0147496e6a4c360a12354eae7e77c1c3cce43ee0cbd8363b057ff/operationStatuses/08584112299876229126\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4951296/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4951296/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j4951296\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report a3c803d5-d069-4edc-a39b-9f91145ca0cc:9/25/2026 11:10:16 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T23:12:33.412Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:12:34.442Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:12:35.811Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=73d52e5b480949f96d0b2ae52647f133586c21c539cb602cbec1e172614eff39 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4951296%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F73d52e5b480949f96d0b2ae52647f133586c21c539cb602cbec1e172614eff39\ntime=2026-09-25T23:19:21.447Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103612411738066944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7183,"pr_state":"closed","pr_sha":"36468bb672ac10f5864c6303a21773ff0e0c3fdb","final_merged_sha":"30a9a5e8593d1558577c9c639b734ef2df1555b8","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T22:27:36Z","started_at":"2026-09-25T22:27:36Z","completed_at":"2026-09-25T23:19:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T23:30:10.480373479Z","timing_metadata_checked_at":"2026-09-25T23:30:10.480373479Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T23:30:10.484710856Z","region_metadata_checked_at":"2026-09-25T23:30:10.484710856Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e40a92f4c98a224d1ed3755491476406fe1ebafb467425277241f32b3fa32731","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103612411738066944","occurred_at":"2026-09-25T22:27:36Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T22:44:02.807Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:44:04.658Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:44:05.624Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8392751d1a1073fcaef6163fdf7bcb8aad59e2453ad8ab27d90e969bb8606250 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8066944%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8392751d1a1073fcaef6163fdf7bcb8aad59e2453ad8ab27d90e969bb8606250\ntime=2026-09-25T22:48:06.937Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8066944/providers/Microsoft.Resources/deployments/8392751d1a1073fcaef6163fdf7bcb8aad59e2453ad8ab27d90e969bb8606250/operationStatuses/08584112306404788422\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8066944/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8066944/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8066944\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report bf065e78-5bcb-4643-98cb-08edae0ce025:9/25/2026 10:45:09 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:50:06.938Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:50:07.547Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:50:08.500Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5a9593d54593dc322c59907a6816979573198c34e3c2436ef231af7d7ef03fa1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8066944%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5a9593d54593dc322c59907a6816979573198c34e3c2436ef231af7d7ef03fa1\ntime=2026-09-25T23:06:12.703Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8066944/providers/Microsoft.Resources/deployments/5a9593d54593dc322c59907a6816979573198c34e3c2436ef231af7d7ef03fa1/operationStatuses/08584112302775421251\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8066944/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8066944/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8066944\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report fad5d579-4dec-4d84-b175-8350099d82ca:9/25/2026 11:05:56 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T23:08:12.704Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:08:13.662Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:08:14.454Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=3ddff724d7ece8cee63d8c21a350143b8534e9c87a8e32e598621b0df2c285f7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8066944%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3ddff724d7ece8cee63d8c21a350143b8534e9c87a8e32e598621b0df2c285f7\ntime=2026-09-25T23:14:02.508Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103610246076305408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7172,"pr_state":"closed","pr_sha":"2bbccece3c6c89e452cce57250e8ff9917d58772","final_merged_sha":"2bbccece3c6c89e452cce57250e8ff9917d58772","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T22:19:00Z","started_at":"2026-09-25T22:19:00Z","completed_at":"2026-09-25T23:13:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T23:30:10.20499601Z","timing_metadata_checked_at":"2026-09-25T23:30:10.20499601Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T23:30:10.207853795Z","region_metadata_checked_at":"2026-09-25T23:30:10.207853795Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f44e46807ea6732f7f4023a72a3faa95c3184e534caec33584898016cebe8e97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103610246076305408","occurred_at":"2026-09-25T22:19:00Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T22:39:04.662Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:39:06.412Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:39:07.373Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=22513326c64b38f225e9dbbf4ea7b709e545583a5df899cc02d1b7a480bdff44 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6305408%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F22513326c64b38f225e9dbbf4ea7b709e545583a5df899cc02d1b7a480bdff44\ntime=2026-09-25T22:43:38.168Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6305408/providers/Microsoft.Resources/deployments/22513326c64b38f225e9dbbf4ea7b709e545583a5df899cc02d1b7a480bdff44/operationStatuses/08584112309387000539\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6305408/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6305408/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6305408\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 508fb5dc-424e-4300-babc-d46ad253ec14:9/25/2026 10:40:09 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:45:38.168Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:45:39.245Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:45:40.298Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=23363d25bacc79be095e8b432161b9765eed12858d9d836d6d890e8058ea566f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6305408%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F23363d25bacc79be095e8b432161b9765eed12858d9d836d6d890e8058ea566f\ntime=2026-09-25T23:01:13.983Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6305408/providers/Microsoft.Resources/deployments/23363d25bacc79be095e8b432161b9765eed12858d9d836d6d890e8058ea566f/operationStatuses/08584112305457657465\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6305408/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6305408/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6305408\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 79f31225-2499-434a-be9e-2ef7ca95e55c:9/25/2026 11:00:58 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T23:03:13.986Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:03:14.652Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:03:15.407Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=68da6d4ed17cebc732fdb2b7c667341dacc23f8f82a0d79cb096c88196c360f2 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6305408%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F68da6d4ed17cebc732fdb2b7c667341dacc23f8f82a0d79cb096c88196c360f2\ntime=2026-09-25T23:09:04.307Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103610217940914176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7176,"pr_state":"closed","pr_sha":"a90ad015f6e70d63790d33a15a5294c05dbf614b","final_merged_sha":"a90ad015f6e70d63790d33a15a5294c05dbf614b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T22:18:52Z","started_at":"2026-09-25T22:18:52Z","completed_at":"2026-09-25T23:18:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T23:30:10.341114702Z","timing_metadata_checked_at":"2026-09-25T23:30:10.341114702Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T23:30:10.343729689Z","region_metadata_checked_at":"2026-09-25T23:30:10.343729689Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fa103c3d9b7bea029203a331d48deb142fd35b372134a7be199cfdc054085ea8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103610217940914176","occurred_at":"2026-09-25T22:18:52Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T22:42:35.008Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:42:36.503Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:42:37.205Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2d1d2b2b69716b8a7c26c037fe69f369f522c795ef40ec0114288c5e566c8b9f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0914176%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2d1d2b2b69716b8a7c26c037fe69f369f522c795ef40ec0114288c5e566c8b9f\ntime=2026-09-25T22:49:38.522Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0914176/providers/Microsoft.Resources/deployments/2d1d2b2b69716b8a7c26c037fe69f369f522c795ef40ec0114288c5e566c8b9f/operationStatuses/08584112307287321237\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0914176/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0914176/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j0914176\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 135c7441-7074-48eb-880c-4b550f01e484:9/25/2026 10:43:45 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:51:38.523Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:51:39.519Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:51:40.392Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c784ffbc3965a0ac80c0c5fc963933d407bab3f0da4a12e8e195ac3c02fd2c72 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0914176%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc784ffbc3965a0ac80c0c5fc963933d407bab3f0da4a12e8e195ac3c02fd2c72\ntime=2026-09-25T23:07:43.802Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0914176/providers/Microsoft.Resources/deployments/c784ffbc3965a0ac80c0c5fc963933d407bab3f0da4a12e8e195ac3c02fd2c72/operationStatuses/08584112301854673287\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0914176/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0914176/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j0914176\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 02165fbb-029a-40d7-8c06-ff7be8bfbcfa:9/25/2026 11:07:17 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T23:09:43.808Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:09:44.517Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:09:45.241Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c98dccbed27b434c3e613ba04fc9b2e7cac35c5feef1c197266857256ecba8ec portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0914176%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc98dccbed27b434c3e613ba04fc9b2e7cac35c5feef1c197266857256ecba8ec\ntime=2026-09-25T23:12:34.789Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103610216275775488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7157,"pr_state":"closed","pr_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","final_merged_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T22:18:50Z","started_at":"2026-09-25T22:18:50Z","completed_at":"2026-09-25T23:22:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T23:41:02.845348908Z","timing_metadata_checked_at":"2026-09-25T23:41:02.845348908Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T23:41:02.849132296Z","region_metadata_checked_at":"2026-09-25T23:41:02.849132296Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d5968d7fb8000ad323044312b591e8ad3c7189caf70ad45393fb8ea1bb3a59a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103610216275775488","occurred_at":"2026-09-25T22:18:50Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T22:48:02.460Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:48:04.622Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:48:06.194Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c571686fbc3bdbc1ba4cdff3f733c8404b545130afbfdc80dc378a3358f98344 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5775488%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc571686fbc3bdbc1ba4cdff3f733c8404b545130afbfdc80dc378a3358f98344\ntime=2026-09-25T22:50:37.350Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5775488/providers/Microsoft.Resources/deployments/c571686fbc3bdbc1ba4cdff3f733c8404b545130afbfdc80dc378a3358f98344/operationStatuses/08584112304001342372\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5775488/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5775488/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5775488\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 03a3616e-4552-4673-b10a-725d152577d8:9/25/2026 10:49:19 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:52:37.352Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:52:43.361Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:52:44.682Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2ecfa98cb5685d521a91530afec8d3664ec919e63a423d71effba96b010c4a19 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5775488%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2ecfa98cb5685d521a91530afec8d3664ec919e63a423d71effba96b010c4a19\ntime=2026-09-25T23:08:50.347Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5775488/providers/Microsoft.Resources/deployments/2ecfa98cb5685d521a91530afec8d3664ec919e63a423d71effba96b010c4a19/operationStatuses/08584112301216608899\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5775488/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5775488/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5775488\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 4cbfeeb1-cc45-42b5-849c-34ed8e6ef378:9/25/2026 11:08:16 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T23:10:50.351Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:10:51.469Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:10:52.845Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=6d424b58a45afa08172d3f7203b2b3cc794f855f7c88b5a71f22be9bbd3bafeb portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5775488%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6d424b58a45afa08172d3f7203b2b3cc794f855f7c88b5a71f22be9bbd3bafeb\ntime=2026-09-25T23:18:02.138Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103610212916137984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T22:18:48Z","started_at":"2026-09-25T22:18:48Z","completed_at":"2026-09-25T23:17:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T23:30:10.055281688Z","timing_metadata_checked_at":"2026-09-25T23:30:10.055281688Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T23:30:10.05863867Z","region_metadata_checked_at":"2026-09-25T23:30:10.05863867Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3c775e3356eb7ae137888db283fd5de6ef0118552997af4efa661affb2105123","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103610212916137984","occurred_at":"2026-09-25T22:18:48Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T22:42:34.675Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:42:36.368Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:42:37.146Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=7e9a9787e94da9b27ffb95c663a5d62a6bf7e8f195427a088122f47c5126b6c6 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6137984%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F7e9a9787e94da9b27ffb95c663a5d62a6bf7e8f195427a088122f47c5126b6c6\ntime=2026-09-25T22:47:38.252Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6137984/providers/Microsoft.Resources/deployments/7e9a9787e94da9b27ffb95c663a5d62a6bf7e8f195427a088122f47c5126b6c6/operationStatuses/08584112307288211876\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6137984/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6137984/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6137984\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 320e91e1-78ba-467c-864e-bb08a265f96e:9/25/2026 10:43:48 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:49:38.253Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:49:39.042Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:49:40.058Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=a0ce9612457e5e562d3d375e6da2c7166df6f1dfaad765af0ed5b52ba3f7bd7a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6137984%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fa0ce9612457e5e562d3d375e6da2c7166df6f1dfaad765af0ed5b52ba3f7bd7a\ntime=2026-09-25T23:05:44.146Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6137984/providers/Microsoft.Resources/deployments/a0ce9612457e5e562d3d375e6da2c7166df6f1dfaad765af0ed5b52ba3f7bd7a/operationStatuses/08584112303060515270\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6137984/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6137984/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6137984\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 002efd10-8e61-4d56-b716-3223e9863285:9/25/2026 11:05:19 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T23:07:44.151Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T23:07:44.966Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T23:07:46.033Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=a10ced99090f5b3a46539b7c2723eb7214d475db9d3b6eba104ae6c17949b809 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6137984%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fa10ced99090f5b3a46539b7c2723eb7214d475db9d3b6eba104ae6c17949b809\ntime=2026-09-25T23:12:34.332Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103605913662263296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"1f52bad80d3779b1af0809af78b4a5b351d87b4c","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T22:01:44Z","started_at":"2026-09-25T22:01:44Z","completed_at":"2026-09-25T22:47:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:58:48.430804863Z","timing_metadata_checked_at":"2026-09-25T22:58:48.430804863Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:58:48.433871452Z","region_metadata_checked_at":"2026-09-25T22:58:48.433871452Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"39a9fba95b1614f8c18219d316d9ba5baaf410f07ad40af0235f6a9be766fae4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103605913662263296","occurred_at":"2026-09-25T22:01:44Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T22:13:13.192Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:13:14.657Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:13:15.731Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=47f17644c7eb5d74ad5721fc987a4fbf0c6d0440cc2e336a3d936dd82387df7d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2263296%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F47f17644c7eb5d74ad5721fc987a4fbf0c6d0440cc2e336a3d936dd82387df7d\ntime=2026-09-25T22:15:46.354Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2263296/providers/Microsoft.Resources/deployments/47f17644c7eb5d74ad5721fc987a4fbf0c6d0440cc2e336a3d936dd82387df7d/operationStatuses/08584112324904058374\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2263296/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2263296/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j2263296\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 605fddca-b452-40f1-8fe5-5d8677045278:9/25/2026 10:14:18 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:17:46.359Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:17:46.955Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:17:47.776Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=06ff40c0f36f5313b1a0b742b62c8868544f40443271212736b81c343ab549d4 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2263296%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F06ff40c0f36f5313b1a0b742b62c8868544f40443271212736b81c343ab549d4\ntime=2026-09-25T22:34:21.322Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2263296/providers/Microsoft.Resources/deployments/06ff40c0f36f5313b1a0b742b62c8868544f40443271212736b81c343ab549d4/operationStatuses/08584112322182337284\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2263296/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2263296/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j2263296\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report c9ce8d30-855b-4007-a894-9e88cdcfaa5a:9/25/2026 10:33:36 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:36:21.322Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:36:21.937Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:36:22.869Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d84b45ccea9ce545c8b61c2e095eab22ed2f82584faab6cf6c94447a39a39093 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2263296%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd84b45ccea9ce545c8b61c2e095eab22ed2f82584faab6cf6c94447a39a39093\ntime=2026-09-25T22:43:12.836Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103603168511266816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7185,"pr_state":"closed","pr_sha":"e06c0760061b363bb7d52b563a23f1cb93be3b02","final_merged_sha":"82a0c5b832cb3e6259e0c6c1090e6105a0f71753","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T21:50:51Z","started_at":"2026-09-25T21:50:51Z","completed_at":"2026-09-25T22:43:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:58:48.535560668Z","timing_metadata_checked_at":"2026-09-25T22:58:48.535560668Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:58:48.538465657Z","region_metadata_checked_at":"2026-09-25T22:58:48.538465657Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3a6dfc619a7d5b412143ec985cca7a7ca099b12a06a8bbdde6e5409a36ac4e9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103603168511266816","occurred_at":"2026-09-25T21:50:51Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T22:06:33.419Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:06:35.080Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:06:35.786Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2d06065552fdb1b58f05c809443bca505130823f2921fb93585cc7903c21fc87 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1266816%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2d06065552fdb1b58f05c809443bca505130823f2921fb93585cc7903c21fc87\ntime=2026-09-25T22:09:06.457Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1266816/providers/Microsoft.Resources/deployments/2d06065552fdb1b58f05c809443bca505130823f2921fb93585cc7903c21fc87/operationStatuses/08584112328901756004\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1266816/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1266816/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1266816\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 1737ff8e-1860-417e-b191-a23050e1d797:9/25/2026 10:07:44 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:11:06.460Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:11:07.274Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:11:08.304Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=aeacaf0120ac07c1ae7a5050f49f02c6463330e79998b2b8372698e977cc301d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1266816%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Faeacaf0120ac07c1ae7a5050f49f02c6463330e79998b2b8372698e977cc301d\ntime=2026-09-25T22:27:11.645Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1266816/providers/Microsoft.Resources/deployments/aeacaf0120ac07c1ae7a5050f49f02c6463330e79998b2b8372698e977cc301d/operationStatuses/08584112326179323078\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1266816/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1266816/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1266816\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report a96f1370-87fe-40e7-a3b2-d48dbac765d0:9/25/2026 10:27:00 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:29:11.653Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:29:12.399Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:29:13.120Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9e44aabeb08ff4c13bc1e8b061b2dee4da754e6ee3317bd096b510a6d2322d40 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1266816%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9e44aabeb08ff4c13bc1e8b061b2dee4da754e6ee3317bd096b510a6d2322d40\ntime=2026-09-25T22:36:33.106Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103599075277410304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7074,"pr_state":"open","pr_sha":"183179ac11c500b20a5a115a52bcd2be4e647fd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T21:34:37Z","started_at":"2026-09-25T21:34:37Z","completed_at":"2026-09-25T22:37:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:47:56.115153296Z","timing_metadata_checked_at":"2026-09-25T22:47:56.115153296Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:47:56.118387886Z","region_metadata_checked_at":"2026-09-25T22:47:56.118387886Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0b5aa28a7babb4eb8599494e15149cb10e9c36fe7d1d8518c0bd7fbf7695778e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103599075277410304","occurred_at":"2026-09-25T21:34:37Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T22:02:18.569Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:02:20.557Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:02:21.831Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=99472b4338634409d6b9001badf0716037c3b1e5a8a8718e702b6cabee1db78a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7410304%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F99472b4338634409d6b9001badf0716037c3b1e5a8a8718e702b6cabee1db78a\ntime=2026-09-25T22:05:23.027Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7410304/providers/Microsoft.Resources/deployments/99472b4338634409d6b9001badf0716037c3b1e5a8a8718e702b6cabee1db78a/operationStatuses/08584112331444971528\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7410304/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7410304/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j7410304\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report c68699e0-a25d-431e-a8b9-31ca1d183f13:9/25/2026 10:03:38 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:07:23.029Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:07:23.904Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:07:25.315Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=01a4a674c001f04222d9e2e14ef00749f4174832c218d467c2f7129891d0b2b1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7410304%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F01a4a674c001f04222d9e2e14ef00749f4174832c218d467c2f7129891d0b2b1\ntime=2026-09-25T22:23:58.812Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7410304/providers/Microsoft.Resources/deployments/01a4a674c001f04222d9e2e14ef00749f4174832c218d467c2f7129891d0b2b1/operationStatuses/08584112328411102221\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7410304/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7410304/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j7410304\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 3a39ba3d-d3bc-4384-ac5d-97f056cd246a:9/25/2026 10:23:20 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:25:58.813Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:25:59.680Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:26:01.058Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=df302a76e783f119f78e93d8858876ecc7bf7d4138d928e0cdc955cddb7705a7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7410304%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fdf302a76e783f119f78e93d8858876ecc7bf7d4138d928e0cdc955cddb7705a7\ntime=2026-09-25T22:32:18.255Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595508659392512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7164,"pr_state":"closed","pr_sha":"653ff4451ef4e6d7031565b7f3e25b8a6be9a485","final_merged_sha":"653ff4451ef4e6d7031565b7f3e25b8a6be9a485","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T21:20:26Z","started_at":"2026-09-25T21:20:26Z","completed_at":"2026-09-25T22:30:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:47:57.795840363Z","timing_metadata_checked_at":"2026-09-25T22:47:57.795840363Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:47:57.799524852Z","region_metadata_checked_at":"2026-09-25T22:47:57.799524852Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ddf61ae33cb19f9d843cd4eb3bdb4f2ffd4c12f76de5b03fbe8c3ccf84f306f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595508659392512","occurred_at":"2026-09-25T21:20:26Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T21:53:44.211Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:53:45.681Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:53:46.865Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=1a50ac725f7d9cfdd91211f218df7feaa93262974760751878ccc979c7d35699 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9392512%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F1a50ac725f7d9cfdd91211f218df7feaa93262974760751878ccc979c7d35699\ntime=2026-09-25T21:56:17.327Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9392512/providers/Microsoft.Resources/deployments/1a50ac725f7d9cfdd91211f218df7feaa93262974760751878ccc979c7d35699/operationStatuses/08584112336591413325\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9392512/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9392512/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9392512\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report c1bdbebf-ee11-4a60-b1bf-045820ef45c3:9/25/2026 9:55:06 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T21:58:17.327Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:58:18.082Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:58:19.073Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=509a65fffa676ed3e77df72b9ba04f18c21701b9cc17c5e678460a764ad6e1dd portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9392512%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F509a65fffa676ed3e77df72b9ba04f18c21701b9cc17c5e678460a764ad6e1dd\ntime=2026-09-25T22:14:21.977Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9392512/providers/Microsoft.Resources/deployments/509a65fffa676ed3e77df72b9ba04f18c21701b9cc17c5e678460a764ad6e1dd/operationStatuses/08584112333870095207\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9392512/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9392512/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9392512\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 3a458705-ca10-40ce-8905-95802398ef90:9/25/2026 10:14:00 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:16:21.982Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:16:22.555Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:16:23.408Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=837be6f8d855476221dd063e7bc507449387eb5521dd1b56a2b4402308941fa9 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9392512%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F837be6f8d855476221dd063e7bc507449387eb5521dd1b56a2b4402308941fa9\ntime=2026-09-25T22:23:43.857Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595507854086144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7165,"pr_state":"open","pr_sha":"4ada4bfca590fd84b0b6be17befd2127f1df09bf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T21:20:26Z","started_at":"2026-09-25T21:20:26Z","completed_at":"2026-09-25T22:25:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:37:23.349448153Z","timing_metadata_checked_at":"2026-09-25T22:37:23.349448153Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:37:23.352856742Z","region_metadata_checked_at":"2026-09-25T22:37:23.352856742Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4d139799b7f5fb883c5cbf5c413e038773dd81f15ea1536fca4c74d1ed2cb43b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595507854086144","occurred_at":"2026-09-25T21:20:26Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T21:50:06.120Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:50:07.873Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:50:09.356Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d03e1a292c576dac5c8301caa55d975be5e38b1917b455138b69cec424f61b0a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4086144%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd03e1a292c576dac5c8301caa55d975be5e38b1917b455138b69cec424f61b0a\ntime=2026-09-25T21:54:11.036Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4086144/providers/Microsoft.Resources/deployments/d03e1a292c576dac5c8301caa55d975be5e38b1917b455138b69cec424f61b0a/operationStatuses/08584112338769505514\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4086144/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4086144/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j4086144\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report db5912b1-70e5-4b02-b80d-1a857a65da86:9/25/2026 9:51:18 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T21:56:11.036Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:56:12.060Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:56:13.437Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=617c40614305f86e9715e915c937241a78354297c7da3c2263549542f81800a1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4086144%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F617c40614305f86e9715e915c937241a78354297c7da3c2263549542f81800a1\ntime=2026-09-25T22:12:17.854Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4086144/providers/Microsoft.Resources/deployments/617c40614305f86e9715e915c937241a78354297c7da3c2263549542f81800a1/operationStatuses/08584112335129278111\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4086144/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4086144/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j4086144\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 57de15b2-2d4f-475f-9107-0f3a5713ed4c:9/25/2026 10:11:41 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:14:17.855Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:14:18.968Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:14:20.352Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=a2bda948e9a8403d3aa3939447fe1f8a73bab008899e150200c842a195fa44d9 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4086144%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fa2bda948e9a8403d3aa3939447fe1f8a73bab008899e150200c842a195fa44d9\ntime=2026-09-25T22:20:05.862Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595503156465664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T21:20:25Z","started_at":"2026-09-25T21:20:25Z","completed_at":"2026-09-25T22:17:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:37:23.222391663Z","timing_metadata_checked_at":"2026-09-25T22:37:23.222391663Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:37:23.225387754Z","region_metadata_checked_at":"2026-09-25T22:37:23.225387754Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"16e1b3e7cdcaa728f3166009b9960c2e8cf97108ca28bc808debe83ccb680702","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595503156465664","occurred_at":"2026-09-25T21:20:25Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T21:42:43.807Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:42:45.430Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:42:46.562Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=fd22dde098a5977bc033224e5904eddedecbb3b77359221bf798eee88eefbfef portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6465664%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ffd22dde098a5977bc033224e5904eddedecbb3b77359221bf798eee88eefbfef\ntime=2026-09-25T21:46:47.293Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6465664/providers/Microsoft.Resources/deployments/fd22dde098a5977bc033224e5904eddedecbb3b77359221bf798eee88eefbfef/operationStatuses/08584112343196674564\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6465664/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6465664/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6465664\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report c2118591-65d0-48d5-96f0-fcbde65124ef:9/25/2026 9:43:53 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T21:48:47.293Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:48:48.055Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:48:48.778Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2e0d1b1055acaac6595d75f50f703569d0ef53962607e7f387f34693ca5d52ef portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6465664%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2e0d1b1055acaac6595d75f50f703569d0ef53962607e7f387f34693ca5d52ef\ntime=2026-09-25T22:04:52.159Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6465664/providers/Microsoft.Resources/deployments/2e0d1b1055acaac6595d75f50f703569d0ef53962607e7f387f34693ca5d52ef/operationStatuses/08584112339571275630\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6465664/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6465664/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6465664\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report eb53c746-b5e4-4b24-b97d-d7a9feae6be4:9/25/2026 10:04:45 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:06:52.160Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:06:52.883Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:06:53.544Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=49f924d9a201a5b3d73d60b15c05c9dd3e440f669a816da24153cbc2bcc58bf7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6465664%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F49f924d9a201a5b3d73d60b15c05c9dd3e440f669a816da24153cbc2bcc58bf7\ntime=2026-09-25T22:12:43.524Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595459770585088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7142,"pr_state":"closed","pr_sha":"cfda26d7698257f0afa1b25f95e39ab26e8f37ef","final_merged_sha":"cfda26d7698257f0afa1b25f95e39ab26e8f37ef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T21:20:15Z","started_at":"2026-09-25T21:20:15Z","completed_at":"2026-09-25T22:28:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:47:56.28400927Z","timing_metadata_checked_at":"2026-09-25T22:47:56.28400927Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:47:56.288750455Z","region_metadata_checked_at":"2026-09-25T22:47:56.288750455Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3329fce7fcb594ede0827bfdf08e0f9287cf536fd1ab0c9a90669df40f46f573","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595459770585088","occurred_at":"2026-09-25T21:20:15Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T21:52:20.697Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:52:22.653Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:52:24.076Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cd0f4ad3c65380d2b50f0791d1d36eaf46a9a6c5ca63fc3b3b105d614bb21f14 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0585088%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcd0f4ad3c65380d2b50f0791d1d36eaf46a9a6c5ca63fc3b3b105d614bb21f14\ntime=2026-09-25T21:56:25.767Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0585088/providers/Microsoft.Resources/deployments/cd0f4ad3c65380d2b50f0791d1d36eaf46a9a6c5ca63fc3b3b105d614bb21f14/operationStatuses/08584112337423399790\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0585088/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0585088/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j0585088\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 58f0013c-5b53-4034-afa7-3ec4cdef6cf4:9/25/2026 9:53:39 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T21:58:25.768Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:58:26.625Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:58:28.251Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=0759dd09b87a5805e985de9142dcb7f8b3815453c07bbd01b3ee4bd26cac9730 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0585088%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F0759dd09b87a5805e985de9142dcb7f8b3815453c07bbd01b3ee4bd26cac9730\ntime=2026-09-25T22:15:03.161Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0585088/providers/Microsoft.Resources/deployments/0759dd09b87a5805e985de9142dcb7f8b3815453c07bbd01b3ee4bd26cac9730/operationStatuses/08584112333782469751\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0585088/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0585088/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j0585088\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 057e74d5-5301-4870-9f54-cb8c8fb9c4e0:9/25/2026 10:14:27 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:17:03.167Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:17:04.308Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:17:05.750Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=222be4a6cc52572ba400b8cf3dbc2676b69280328886a585ad459067ddbb0c5c portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0585088%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F222be4a6cc52572ba400b8cf3dbc2676b69280328886a585ad459067ddbb0c5c\ntime=2026-09-25T22:22:20.287Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595446143291392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T21:20:11Z","started_at":"2026-09-25T21:20:11Z","completed_at":"2026-09-25T22:36:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:47:55.904533152Z","timing_metadata_checked_at":"2026-09-25T22:47:55.904533152Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:47:55.907806441Z","region_metadata_checked_at":"2026-09-25T22:47:55.907806441Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fb11a464eddef7edf6eeceb9a7a65e641642f8f2eb765d6ee5a7d3c157fa62c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595446143291392","occurred_at":"2026-09-25T21:20:11Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T22:01:43.704Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:01:45.608Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:01:46.776Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2b02570b2f288f755b2262ae572ae2d502ac1159f0377dcfa2a5df1b1eb88050 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3291392%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2b02570b2f288f755b2262ae572ae2d502ac1159f0377dcfa2a5df1b1eb88050\ntime=2026-09-25T22:04:47.619Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3291392/providers/Microsoft.Resources/deployments/2b02570b2f288f755b2262ae572ae2d502ac1159f0377dcfa2a5df1b1eb88050/operationStatuses/08584112331794344085\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3291392/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3291392/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3291392\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 9e278c13-59ad-47b0-b079-9caa20281fb2:9/25/2026 10:03:06 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:06:47.621Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:06:48.437Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:06:49.907Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=977a91a1f627a8f114578a64d64b6608dbc9b4132d84d55dec59555e661a3508 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3291392%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F977a91a1f627a8f114578a64d64b6608dbc9b4132d84d55dec59555e661a3508\ntime=2026-09-25T22:23:23.704Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3291392/providers/Microsoft.Resources/deployments/977a91a1f627a8f114578a64d64b6608dbc9b4132d84d55dec59555e661a3508/operationStatuses/08584112328764640322\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3291392/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3291392/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3291392\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report eb5ab2b2-1a03-4555-82cf-929f5ddb6563:9/25/2026 10:22:48 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:25:23.709Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:25:24.611Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:25:26.034Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=0119ef8bd7ce729d8931ebfd0837fa7168dec93c713688f4f1dfd006dc22a851 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3291392%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F0119ef8bd7ce729d8931ebfd0837fa7168dec93c713688f4f1dfd006dc22a851\ntime=2026-09-25T22:31:43.284Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595427583496192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7168,"pr_state":"closed","pr_sha":"438a454514812b6dc5eb79970e7dce00ce113844","final_merged_sha":"438a454514812b6dc5eb79970e7dce00ce113844","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T21:20:03Z","started_at":"2026-09-25T21:20:03Z","completed_at":"2026-09-25T22:34:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:47:58.189513738Z","timing_metadata_checked_at":"2026-09-25T22:47:58.189513738Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:47:58.197084714Z","region_metadata_checked_at":"2026-09-25T22:47:58.197084714Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"33bb70066504caf46462bd6f83a53dffc3084481f0b101741a14e9f3e07a2000","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595427583496192","occurred_at":"2026-09-25T21:20:03Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T21:58:43.521Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:58:47.335Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:58:48.679Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=26188bc42aa9c7e6645f2fcd4b05c95ba7b609b45891905f3be3a6a29e0e5268 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3496192%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F26188bc42aa9c7e6645f2fcd4b05c95ba7b609b45891905f3be3a6a29e0e5268\ntime=2026-09-25T22:02:20.196Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3496192/providers/Microsoft.Resources/deployments/26188bc42aa9c7e6645f2fcd4b05c95ba7b609b45891905f3be3a6a29e0e5268/operationStatuses/08584112333576052502\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3496192/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3496192/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3496192\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report c23fef9b-1941-46ed-ac66-6952f36d4e8e:9/25/2026 10:00:08 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:04:20.205Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:04:26.213Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:04:27.558Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f43cd742e9d6dc746f2874d92836b6581a930015296112ae9c621cfa96bbf0f8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3496192%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff43cd742e9d6dc746f2874d92836b6581a930015296112ae9c621cfa96bbf0f8\ntime=2026-09-25T22:20:31.934Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3496192/providers/Microsoft.Resources/deployments/f43cd742e9d6dc746f2874d92836b6581a930015296112ae9c621cfa96bbf0f8/operationStatuses/08584112330187400918\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3496192/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3496192/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3496192\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 145375bd-41dd-4413-841f-64089f131e87:9/25/2026 10:20:17 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:22:31.935Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:22:32.999Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:22:34.465Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e861e2ee87ecfaa5d598705703071b17c1a1af9fe4c55d15a2b1b85d40060e6e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3496192%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fe861e2ee87ecfaa5d598705703071b17c1a1af9fe4c55d15a2b1b85d40060e6e\ntime=2026-09-25T22:28:43.299Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595282880008192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"a76a535be60a9334a178a32c44d504929096061e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T21:19:32Z","started_at":"2026-09-25T21:19:32Z","completed_at":"2026-09-25T22:23:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:37:23.493090489Z","timing_metadata_checked_at":"2026-09-25T22:37:23.493090489Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:37:23.495567481Z","region_metadata_checked_at":"2026-09-25T22:37:23.495567481Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a7b1cb37fe36c2601877766ea51cb4dd5e2c9eeb4694622098ad1f85f4da0b8f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595282880008192","occurred_at":"2026-09-25T21:19:32Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T21:46:51.124Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:46:52.924Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:46:54.269Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9dd6cfe4c5cca40ff4140c66b83c8d7e859e838264966d12c46ab47e16965351 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0008192%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9dd6cfe4c5cca40ff4140c66b83c8d7e859e838264966d12c46ab47e16965351\ntime=2026-09-25T21:50:25.624Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0008192/providers/Microsoft.Resources/deployments/9dd6cfe4c5cca40ff4140c66b83c8d7e859e838264966d12c46ab47e16965351/operationStatuses/08584112340720340851\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0008192/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0008192/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j0008192\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 0ee67748-58a8-4d3c-806f-e9955259d88b:9/25/2026 9:48:04 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T21:52:25.633Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:52:26.872Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:52:28.329Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d5f2a87444d32944fcd12dbe9a103edec0a607cad5bfb0b4cacca36c8db97e41 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0008192%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd5f2a87444d32944fcd12dbe9a103edec0a607cad5bfb0b4cacca36c8db97e41\ntime=2026-09-25T22:08:33.327Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0008192/providers/Microsoft.Resources/deployments/d5f2a87444d32944fcd12dbe9a103edec0a607cad5bfb0b4cacca36c8db97e41/operationStatuses/08584112337381035589\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0008192/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0008192/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j0008192\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report b7e86fb3-ed18-496f-a975-f2904e5ffd3f:9/25/2026 10:08:23 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:10:33.334Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:10:34.051Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:10:35.482Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=6bfd52263c9492783ab534eaca5a1f8395504d120ae1f6d7ca0e958f10cf64b8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0008192%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6bfd52263c9492783ab534eaca5a1f8395504d120ae1f6d7ca0e958f10cf64b8\ntime=2026-09-25T22:16:50.729Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103594698844147712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"be188df89897e2934f949038ba430a4fab081347","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T21:17:13Z","started_at":"2026-09-25T21:17:13Z","completed_at":"2026-09-25T22:18:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:37:23.202164429Z","timing_metadata_checked_at":"2026-09-25T22:37:23.202164429Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:37:23.205271019Z","region_metadata_checked_at":"2026-09-25T22:37:23.205271019Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ccf92fa3395c3ee292e020e7a698809a09f5d2bc782e2ab7b8ff0f6d005d94f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103594698844147712","occurred_at":"2026-09-25T21:17:13Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T21:42:57.209Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:42:59.058Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:43:00.565Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=6e75299a25eec3728f1167bcbbda424357f89f99cee8ea6a1ac46f7a3f4c980c portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4147712%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6e75299a25eec3728f1167bcbbda424357f89f99cee8ea6a1ac46f7a3f4c980c\ntime=2026-09-25T21:45:31.708Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4147712/providers/Microsoft.Resources/deployments/6e75299a25eec3728f1167bcbbda424357f89f99cee8ea6a1ac46f7a3f4c980c/operationStatuses/08584112343058335582\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4147712/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4147712/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j4147712\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 838327d2-2363-4fb6-bf56-5d7dcb53235c:9/25/2026 9:44:11 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T21:47:31.714Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:47:37.446Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:47:38.962Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=4d4b089cd23c38c9356171223251164b75e7334bc676ddde23fc5de5b3fd7dc7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4147712%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4d4b089cd23c38c9356171223251164b75e7334bc676ddde23fc5de5b3fd7dc7\ntime=2026-09-25T22:03:44.271Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4147712/providers/Microsoft.Resources/deployments/4d4b089cd23c38c9356171223251164b75e7334bc676ddde23fc5de5b3fd7dc7/operationStatuses/08584112340275417299\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4147712/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4147712/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j4147712\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 10c1522f-953b-459c-9f55-808545dc01ec:9/25/2026 10:02:57 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T22:05:44.278Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T22:05:45.280Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T22:05:48.383Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d3fc6d296a22e861f508461ee14630c914b6220547a43caa38c09c6bebf492c4 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4147712%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd3fc6d296a22e861f508461ee14630c914b6220547a43caa38c09c6bebf492c4\ntime=2026-09-25T22:12:56.875Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7179/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103589524364857344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7179,"pr_state":"closed","pr_sha":"a6b6948af5457d947d7373a6515542a3c062be40","final_merged_sha":"a6b6948af5457d947d7373a6515542a3c062be40","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T20:56:40Z","started_at":"2026-09-25T20:56:40Z","completed_at":"2026-09-25T23:04:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T23:19:55.647103412Z","timing_metadata_checked_at":"2026-09-25T23:19:55.647103412Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T23:19:55.650590901Z","region_metadata_checked_at":"2026-09-25T23:19:55.650590901Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103587716158459904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7146,"pr_state":"closed","pr_sha":"34a27988a2da6ca8b624fbab48af075e636b7eb6","final_merged_sha":"34a27988a2da6ca8b624fbab48af075e636b7eb6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T20:49:28Z","started_at":"2026-09-25T20:49:28Z","completed_at":"2026-09-25T22:58:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T23:09:07.129578047Z","timing_metadata_checked_at":"2026-09-25T23:09:07.129578047Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T23:09:07.132195139Z","region_metadata_checked_at":"2026-09-25T23:09:07.132195139Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103584488377290752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7104,"pr_state":"closed","pr_sha":"f0ac812f608c88c23fcd4648e295dfb77018622d","final_merged_sha":"a2e35daa20b37841bf1d2f73e88d95554cdc5664","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T20:36:39Z","started_at":"2026-09-25T20:36:39Z","completed_at":"2026-09-25T22:50:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T23:09:06.812968181Z","timing_metadata_checked_at":"2026-09-25T23:09:06.812968181Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-09-25T23:09:06.816219572Z","region_metadata_checked_at":"2026-09-26T00:12:25.463016287Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"99492807df5915610cd651d6eaf51060988022c5e060fea854b6b2ced89b4795","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103584488377290752","occurred_at":"2026-09-25T20:36:39Z","failed_at":"e2e","signature_id":"dd6c8bcde2b0624cac2879270fee01eca0a67b3bbceee5d0a34b90be6db52b16","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002da48c0\u003e: \n DNS for route host agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002da48c0\u003e: \n DNS for route host agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579630278545408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T20:17:21Z","started_at":"2026-09-25T20:17:21Z","completed_at":"2026-09-25T22:37:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:47:58.387292822Z","timing_metadata_checked_at":"2026-09-25T22:47:58.387292822Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:47:58.390517012Z","region_metadata_checked_at":"2026-09-25T22:47:58.390517012Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"19aa3469e04b8d90f3fdd358acf02d3184a082ef6e1e5d2b6058382c4eed3883","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579630278545408","occurred_at":"2026-09-25T20:17:21Z","failed_at":"e2e","signature_id":"0c6454c08c8ccd6642c0993a10a72be6b940d4b7c10aa86a4dd03a639e166731","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:286]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d1f800\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:286]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d1f800\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"5a834e3201a29f6fd2e1e6264a80eb206d1b27797b8c87f19c5a7b7373823729","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579630278545408","occurred_at":"2026-09-25T20:17:21Z","failed_at":"alert","signature_id":"f80738046c4e039d573791df16b6cbb481153102a2bb23701d68c69a8a6a851d","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T21:24:49Z\n Ended: 2026-09-25T21:44:50Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j8545408-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t29brjns56m1264ar7oq3gbmbspvafd-t7i2h8r3z4p7q4p\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j8545408-mgmt-1\nNamespace: ocm-arohcpci01-2t29brjns56m1264ar7oq3gbmbspvafd-t7i2h8r3z4p7q4p","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"73eaa560e2fb002c33c8e1d6281de77cf397f3e8316299e6c0d2ab838c47c8e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579630278545408","occurred_at":"2026-09-25T20:17:21Z","failed_at":"alert","signature_id":"c019ffe1f564224958ef7dd4995e2ce1348ec572e7f126e871a8a7be641c251e","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T21:27:50Z\n Ended: 2026-09-25T22:09:49Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j8545408-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t29brjns56m1264ar7oq3gbmbspvafd-t7i2h8r3z4p7q4p\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j8545408-mgmt-1\nNamespace: ocm-arohcpci01-2t29brjns56m1264ar7oq3gbmbspvafd-t7i2h8r3z4p7q4p","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"ef716d478c996a7119bcd103e44767b19b399945f6def5a9864917b40c82fd58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579630278545408","occurred_at":"2026-09-25T20:17:21Z","failed_at":"e2e","signature_id":"04e92aed41dc4397943b4c76c401adb73bdb624b4bd11916358f9767c8bfe9d6","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:397]: failed to pull image after updating containerRegistry to second MI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b92200\u003e: \n VerifyImagePulled(acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal) timed out after 3m0.006s: expected image from acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:\n - pod acr-pull-test-after-update container acr-pull-test: ImagePullBackOff - Back-off pulling image \"acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal:latest: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: eefe0445-30e7-478a-8283-14278233fefa; artifact err: get manifest: build image source: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: c9d8d031-808a-4e35-9c9c-bff18e860fbb\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:397]: failed to pull image after updating containerRegistry to second MI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b92200\u003e: \n VerifyImagePulled(acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal) timed out after 3m0.006s: expected image from acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:\n - pod acr-pull-test-after-update container acr-pull-test: ImagePullBackOff - Back-off pulling image \"acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal:latest: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: eefe0445-30e7-478a-8283-14278233fefa; artifact err: get manifest: build image source: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: c9d8d031-808a-4e35-9c9c-bff18e860fbb\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7128002351c39d7ffe6bc9e97f3e7bea8a864e7268f53656aca162804a977c0a","failure_pattern":"VerifyImagePulled(acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal) timed out after 3m0.006s: expected image from acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:","search_query":"VerifyImagePulled(acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal) timed out after 3m0.006s: expected image from acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","7128002351c39d7ffe6bc9e97f3e7bea8a864e7268f53656aca162804a977c0a","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579624884670464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T20:17:19Z","started_at":"2026-09-25T20:17:19Z","completed_at":"2026-09-25T21:21:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T21:34:14.03677948Z","timing_metadata_checked_at":"2026-09-25T21:34:14.03677948Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T21:34:14.04006357Z","region_metadata_checked_at":"2026-09-25T21:34:14.04006357Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5977844ffc6fb44e6d50ce0836c7cd03d043af2247c395191aeda7c3e539f776","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579624884670464","occurred_at":"2026-09-25T20:17:19Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T20:46:41.564Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:46:43.032Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:46:44.144Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9c63a3f2bb98c0829e583ca8821c38ce090554ad098caaa8feff71da61c0ee1f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4670464%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9c63a3f2bb98c0829e583ca8821c38ce090554ad098caaa8feff71da61c0ee1f\ntime=2026-09-25T20:49:14.657Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4670464/providers/Microsoft.Resources/deployments/9c63a3f2bb98c0829e583ca8821c38ce090554ad098caaa8feff71da61c0ee1f/operationStatuses/08584112376820587100\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4670464/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4670464/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j4670464\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 7619bd65-a7b3-401e-aa91-f3eaa8801578:9/25/2026 8:47:51 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T20:51:14.662Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:51:15.528Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:51:16.258Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8a187738846502bb502f5178f4ab4a65bd57c8f4d854b4d2d8da2ea59d862584 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4670464%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8a187738846502bb502f5178f4ab4a65bd57c8f4d854b4d2d8da2ea59d862584\ntime=2026-09-25T21:07:49.122Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4670464/providers/Microsoft.Resources/deployments/8a187738846502bb502f5178f4ab4a65bd57c8f4d854b4d2d8da2ea59d862584/operationStatuses/08584112374095896474\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4670464/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4670464/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j4670464\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 4dc6c8a1-c2b5-4d76-8719-2a0b40845345:9/25/2026 9:07:02 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T21:09:49.132Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:09:49.945Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:09:50.660Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=04870a07c33243925bed36f12dba141f18927bb4a6f207f0c07294ce5610f4a9 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4670464%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F04870a07c33243925bed36f12dba141f18927bb4a6f207f0c07294ce5610f4a9\ntime=2026-09-25T21:16:41.252Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579604630376448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7157,"pr_state":"closed","pr_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","final_merged_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T20:17:14Z","started_at":"2026-09-25T20:17:14Z","completed_at":"2026-09-25T21:20:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T21:34:14.518918372Z","timing_metadata_checked_at":"2026-09-25T21:34:14.518918372Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T21:34:14.522502561Z","region_metadata_checked_at":"2026-09-25T21:34:14.522502561Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9b201b25f24dc3d4b7ddba224a8ad13136665c637e5055adf215f426f6dcec0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579604630376448","occurred_at":"2026-09-25T20:17:14Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T20:42:57.630Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:42:59.318Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:43:00.591Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=43f97c6ab4bb5233131ff6f5d5f9e41970ca5ed1dcea24eba515091f3904dc4f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0376448%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F43f97c6ab4bb5233131ff6f5d5f9e41970ca5ed1dcea24eba515091f3904dc4f\ntime=2026-09-25T20:46:01.054Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0376448/providers/Microsoft.Resources/deployments/43f97c6ab4bb5233131ff6f5d5f9e41970ca5ed1dcea24eba515091f3904dc4f/operationStatuses/08584112379056358387\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0376448/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0376448/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j0376448\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 029f2547-2d53-4c23-bce3-15f36f6b275e:9/25/2026 8:43:57 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T20:48:01.060Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:48:02.039Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:48:04.631Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=684da2f4a7951a7331c264e297d75f379ec2b20a19736f414ef721dd7d9d546c portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0376448%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F684da2f4a7951a7331c264e297d75f379ec2b20a19736f414ef721dd7d9d546c\ntime=2026-09-25T21:04:37.876Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0376448/providers/Microsoft.Resources/deployments/684da2f4a7951a7331c264e297d75f379ec2b20a19736f414ef721dd7d9d546c/operationStatuses/08584112376029879572\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0376448/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0376448/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j0376448\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 97b8f2c8-a210-440f-a0e8-6042af05ad4a:9/25/2026 9:04:02 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T21:06:37.879Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T21:06:38.631Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T21:06:39.441Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=1086a1ac0665015ea9f533906a517340a12aa697f99d59df95d54135ee822de9 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0376448%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F1086a1ac0665015ea9f533906a517340a12aa697f99d59df95d54135ee822de9\ntime=2026-09-25T21:12:57.297Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103575188418334720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7181,"pr_state":"closed","pr_sha":"03c56fdb2d72507deeabb70f679a4bb80f78c3b6","final_merged_sha":"3ae51eab9b946eb1b94407de9646f52a67175cd0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T19:59:42Z","started_at":"2026-09-25T19:59:42Z","completed_at":"2026-09-25T20:50:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T21:03:12.816082107Z","timing_metadata_checked_at":"2026-09-25T21:03:12.816082107Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T21:03:12.819945495Z","region_metadata_checked_at":"2026-09-25T21:03:12.819945495Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"91f9cdf1302122a04e6d3c56caa4f19903bce0d64fc1a3bdc406d4927af6d3c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103575188418334720","occurred_at":"2026-09-25T19:59:42Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T20:15:07.762Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:15:09.394Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:15:10.306Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=07c3cfce312aa76095bf819171fb5db0280d42daddd3d7f84c2a41113c38b31b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8334720%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F07c3cfce312aa76095bf819171fb5db0280d42daddd3d7f84c2a41113c38b31b\ntime=2026-09-25T20:19:11.006Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8334720/providers/Microsoft.Resources/deployments/07c3cfce312aa76095bf819171fb5db0280d42daddd3d7f84c2a41113c38b31b/operationStatuses/08584112395757644329\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8334720/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8334720/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8334720\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report c8953561-defb-4625-ad48-774d8a740311:9/25/2026 8:16:09 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T20:21:11.006Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:21:11.864Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:21:12.729Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=561574aa1f69813418be96c5c34121f55cdb75a604ffb3e82236308822bc7766 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8334720%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F561574aa1f69813418be96c5c34121f55cdb75a604ffb3e82236308822bc7766\ntime=2026-09-25T20:37:45.383Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8334720/providers/Microsoft.Resources/deployments/561574aa1f69813418be96c5c34121f55cdb75a604ffb3e82236308822bc7766/operationStatuses/08584112392131934543\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8334720/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8334720/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8334720\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 4da4892d-0a87-42a4-8f4d-b334dc12a1ef:9/25/2026 8:37:15 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T20:39:45.387Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:39:46.182Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:39:47.348Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c042d0b6ea7eca096a267838668c337ee86e00d54afccfba51a6125c0aaa956b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8334720%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc042d0b6ea7eca096a267838668c337ee86e00d54afccfba51a6125c0aaa956b\ntime=2026-09-25T20:45:07.439Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103574351302365184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7172,"pr_state":"closed","pr_sha":"2bbccece3c6c89e452cce57250e8ff9917d58772","final_merged_sha":"2bbccece3c6c89e452cce57250e8ff9917d58772","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T19:56:22Z","started_at":"2026-09-25T19:56:22Z","completed_at":"2026-09-25T22:07:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T22:26:32.111590853Z","timing_metadata_checked_at":"2026-09-25T22:26:32.111590853Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T22:26:32.114882343Z","region_metadata_checked_at":"2026-09-25T22:26:32.114882343Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"01febcabe96c4f3134c344a23e35b94a3399b5c3f349d6a92160c3a203badbce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103574351302365184","occurred_at":"2026-09-25T19:56:22Z","failed_at":"e2e","signature_id":"d6768ec87762d85ec18639e021a53550ee7d398e2e064054a52f251bb3dee36d","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000adbee0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qqrdr5ldkfxf\" to finish creating: GET https://rp.j2365184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e499c0b9-b430-40af-9ef1-8fccc02c39c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e499c0b9-b430-40af-9ef1-8fccc02c39c5\",\n \"name\": \"e499c0b9-b430-40af-9ef1-8fccc02c39c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T20:55:06.599822055Z\",\n \"endTime\": \"2026-09-25T21:14:14.361175634Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qqrdr5ldkfxf--managed/providers/Microsoft.Compute/virtualMachines/j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qqrdr5ldkfxf--managed/providers/Microsoft.Compute/virtualMachines/j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000adbee0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qqrdr5ldkfxf\" to finish creating: GET https://rp.j2365184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e499c0b9-b430-40af-9ef1-8fccc02c39c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e499c0b9-b430-40af-9ef1-8fccc02c39c5\",\n \"name\": \"e499c0b9-b430-40af-9ef1-8fccc02c39c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T20:55:06.599822055Z\",\n \"endTime\": \"2026-09-25T21:14:14.361175634Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qqrdr5ldkfxf--managed/providers/Microsoft.Compute/virtualMachines/j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qqrdr5ldkfxf--managed/providers/Microsoft.Compute/virtualMachines/j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dnr4nbhwmf28\" to finish creating: GET https://rp"}},{"row_id":"1900ef1905e553c4222f1ed5dd9bca9e0a36f4199ee9480edaa97cf794d8ac05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103574351302365184","occurred_at":"2026-09-25T19:56:22Z","failed_at":"alert","signature_id":"c658821ff5f7d7b2d63a0e97b55950458ca17f792593dc632182ea487961cf11","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T20:52:51Z\n Ended: 2026-09-25T21:41:55Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j2365184-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t28rm7oll3gsirl5v5655vkmbilcj89-v20261001\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2365184-mgmt-1\nNamespace: ocm-arohcpci01-2t28rm7oll3gsirl5v5655vkmbilcj89-v20261001","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"51dc9d49a2ba0f6125371616fb43518623fc56556afec0ceb18a26c2336c0929","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103574351302365184","occurred_at":"2026-09-25T19:56:22Z","failed_at":"alert","signature_id":"6ddbf27f7eb384cd0a600dae6b3cf441fa8fbe9971573e883b0042de0375bd93","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T20:49:53Z\n Ended: 2026-09-25T21:05:53Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j2365184-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t28rm7oll3gsirl5v5655vkmbilcj89-v20261001\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2365184-mgmt-1\nNamespace: ocm-arohcpci01-2t28rm7oll3gsirl5v5655vkmbilcj89-v20261001\n\nFiring 2:\n State: Resolved\n Started: 2026-09-25T21:08:52Z\n Ended: 2026-09-25T21:16:51Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j2365184-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t28rm7oll3gsirl5v5655vkmbilcj89-v20261001\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2365184-mgmt-1\nNamespace: ocm-arohcpci01-2t28rm7oll3gsirl5v5655vkmbilcj89-v20261001","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"5b2cf047763e7aab7643c476a000940e7ee9f06782224793c2a0d45c1d3d4436","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103574351302365184","occurred_at":"2026-09-25T19:56:22Z","failed_at":"alert","signature_id":"e3101a2b58ad0aba9cc14b18b48bd4af2812352abeb2151468084aebc395f66f","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T20:52:54Z\n Ended: 2026-09-25T21:41:59Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j2365184-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t28rm7oll3gsirl5v5655vkmbilcj89-v20261001\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2365184-mgmt-1\nNamespace: ocm-arohcpci01-2t28rm7oll3gsirl5v5655vkmbilcj89-v20261001","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"e4118276214ce40486025ee5eff860ec675bb9744782b257556933dbeac461de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103574351302365184","occurred_at":"2026-09-25T19:56:22Z","failed_at":"alert","signature_id":"59c796fbf53347a929d1140f136d97e8f215af00e5e2f39835a61c93eeea0c68","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T20:49:54Z\n Ended: 2026-09-25T21:04:52Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j2365184-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t28rm7oll3gsirl5v5655vkmbilcj89-v20261001\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2365184-mgmt-1\nNamespace: ocm-arohcpci01-2t28rm7oll3gsirl5v5655vkmbilcj89-v20261001\n\nFiring 2:\n State: Resolved\n Started: 2026-09-25T21:08:57Z\n Ended: 2026-09-25T21:16:52Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j2365184-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t28rm7oll3gsirl5v5655vkmbilcj89-v20261001\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2365184-mgmt-1\nNamespace: ocm-arohcpci01-2t28rm7oll3gsirl5v5655vkmbilcj89-v20261001","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103572043327868928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7180,"pr_state":"open","pr_sha":"f58afcc1c7298b96467ad2dba099d8a155f0e337","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T19:47:10Z","started_at":"2026-09-25T19:47:10Z","completed_at":"2026-09-25T20:33:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:52:34.26461551Z","timing_metadata_checked_at":"2026-09-25T20:52:34.26461551Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:52:34.269254497Z","region_metadata_checked_at":"2026-09-25T20:52:34.269254497Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9da9afdcb6bb70eb802e877b553c28fece32ab2330092b3e26a3c71e8abae60e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103572043327868928","occurred_at":"2026-09-25T19:47:10Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T19:58:15.008Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T19:58:16.857Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T19:58:17.892Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f657e1ba875cf5a5c564e09bfb94487857818bdc75a7c76dd39827534ec31f3c portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7868928%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff657e1ba875cf5a5c564e09bfb94487857818bdc75a7c76dd39827534ec31f3c\ntime=2026-09-25T20:02:24.848Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7868928/providers/Microsoft.Resources/deployments/f657e1ba875cf5a5c564e09bfb94487857818bdc75a7c76dd39827534ec31f3c/operationStatuses/08584112405883014048\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7868928/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7868928/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j7868928\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 9cb6bfea-0dc6-4236-a7b5-6ca587106c66:9/25/2026 7:59:24 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T20:04:24.848Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:04:25.496Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:04:26.448Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=596f678aa88dbee36e94ffeb65648fcd39f567b192995f1444936f03a41e6f19 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7868928%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F596f678aa88dbee36e94ffeb65648fcd39f567b192995f1444936f03a41e6f19\ntime=2026-09-25T20:19:59.345Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7868928/providers/Microsoft.Resources/deployments/596f678aa88dbee36e94ffeb65648fcd39f567b192995f1444936f03a41e6f19/operationStatuses/08584112402196567184\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7868928/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7868928/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j7868928\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 71edd1e3-0661-4d06-be02-9a76af381bf5:9/25/2026 8:19:44 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T20:21:59.345Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:21:59.958Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:22:00.992Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=eea0d64467b5c04f02b194de8f21b412fe73dcc6d6ddd757bbd3402a0cfd783f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7868928%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Feea0d64467b5c04f02b194de8f21b412fe73dcc6d6ddd757bbd3402a0cfd783f\ntime=2026-09-25T20:28:14.746Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103565386573156352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"e50c6077a32b443e8ccb8d6a01a858b3a4c3d595","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T19:20:35Z","started_at":"2026-09-25T19:20:35Z","completed_at":"2026-09-25T20:19:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:31:33.416724661Z","timing_metadata_checked_at":"2026-09-25T20:31:33.416724661Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:31:33.419517853Z","region_metadata_checked_at":"2026-09-25T20:31:33.419517853Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5b852b37fe8020a745bf8ceb10a4ed1f09bd9293f6eadd9120a7f7d46e8ba43d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103565386573156352","occurred_at":"2026-09-25T19:20:35Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T19:44:59.157Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T19:45:00.843Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T19:45:01.711Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=7031446bc408bbf536d406f3314e0d07f7faad9a6aff2bd819116c81ed9fde27 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3156352%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F7031446bc408bbf536d406f3314e0d07f7faad9a6aff2bd819116c81ed9fde27\ntime=2026-09-25T19:52:03.122Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3156352/providers/Microsoft.Resources/deployments/7031446bc408bbf536d406f3314e0d07f7faad9a6aff2bd819116c81ed9fde27/operationStatuses/08584112413841645419\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3156352/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3156352/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3156352\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report e8910667-d60d-43eb-b394-6291d3afcdf8:9/25/2026 7:46:14 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T19:54:03.125Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T19:54:03.826Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T19:54:04.969Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=da4bbe8728d9405e4a69f67d9e90c14da3cbbc1575a609a8a43c1f456b87bee5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3156352%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fda4bbe8728d9405e4a69f67d9e90c14da3cbbc1575a609a8a43c1f456b87bee5\ntime=2026-09-25T20:10:07.690Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3156352/providers/Microsoft.Resources/deployments/da4bbe8728d9405e4a69f67d9e90c14da3cbbc1575a609a8a43c1f456b87bee5/operationStatuses/08584112408409625094\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3156352/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3156352/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3156352\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report cbdcca33-64aa-4843-af0e-6141e71baa44:9/25/2026 8:09:38 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T20:12:07.697Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:12:08.245Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:12:09.102Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=1842de40bf7d9caae04c3ce5b14acabf28fc5fadf35eab5fea0a753c81379943 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3156352%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F1842de40bf7d9caae04c3ce5b14acabf28fc5fadf35eab5fea0a753c81379943\ntime=2026-09-25T20:14:58.842Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564555174023168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T19:17:26Z","started_at":"2026-09-25T19:17:26Z","completed_at":"2026-09-25T20:18:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:31:33.229243044Z","timing_metadata_checked_at":"2026-09-25T20:31:33.229243044Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:31:33.23374363Z","region_metadata_checked_at":"2026-09-25T20:31:33.23374363Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"af7f7aaea223578e358ca269a74f4abcbb6ebd62f20c6b2856fea4eafa760ddb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564555174023168","occurred_at":"2026-09-25T19:17:26Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T19:42:01.610Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T19:42:03.255Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T19:42:04.004Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=09d07f1a8fc42d0236d7bcc7a5252b313e7e87aed5afc751eb99149ade01fed6 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4023168%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F09d07f1a8fc42d0236d7bcc7a5252b313e7e87aed5afc751eb99149ade01fed6\ntime=2026-09-25T19:45:04.501Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4023168/providers/Microsoft.Resources/deployments/09d07f1a8fc42d0236d7bcc7a5252b313e7e87aed5afc751eb99149ade01fed6/operationStatuses/08584112415619437293\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4023168/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4023168/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j4023168\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 2caf05a7-db95-41ef-891a-27aea277458e:9/25/2026 7:43:10 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T19:47:04.502Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T19:47:05.059Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T19:47:05.970Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f65a5774e15dae8921b43b339c4d1059a50738b399f1268ad02dd460ce228f8a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4023168%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff65a5774e15dae8921b43b339c4d1059a50738b399f1268ad02dd460ce228f8a\ntime=2026-09-25T20:03:08.830Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4023168/providers/Microsoft.Resources/deployments/f65a5774e15dae8921b43b339c4d1059a50738b399f1268ad02dd460ce228f8a/operationStatuses/08584112412601092483\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4023168/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4023168/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j4023168\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report c2a38f62-30a5-4126-827b-ba662adfb1eb:9/25/2026 8:02:42 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T20:05:08.835Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:05:09.583Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:05:11.372Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=4aaac2cc21559c80c5b938b14636d642e5eb83a739fddbb08d378d8b2292a761 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4023168%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4aaac2cc21559c80c5b938b14636d642e5eb83a739fddbb08d378d8b2292a761\ntime=2026-09-25T20:12:01.371Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564545086722048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7174,"pr_state":"closed","pr_sha":"6622cab9ad4d278b9626d29a58f5bbe3a9f5fdc9","final_merged_sha":"756dbd0aea05ea7d18d18b94d48e2ccdb7df1c9c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T19:17:24Z","started_at":"2026-09-25T19:17:24Z","completed_at":"2026-09-25T20:14:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:31:33.797965476Z","timing_metadata_checked_at":"2026-09-25T20:31:33.797965476Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:31:33.801409066Z","region_metadata_checked_at":"2026-09-25T20:31:33.801409066Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a06ea0fdd73a6a474cb41b5346a0b01692bc3b2cb9c6099c0e80f5b7e43c9e0b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564545086722048","occurred_at":"2026-09-25T19:17:24Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T19:38:40.757Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T19:38:42.512Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T19:38:43.316Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c12cafd4f123a2aa0f41691c377a4ee9b6bda880ce2a63c732655640aa124f7e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6722048%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc12cafd4f123a2aa0f41691c377a4ee9b6bda880ce2a63c732655640aa124f7e\ntime=2026-09-25T19:54:15.631Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6722048/providers/Microsoft.Resources/deployments/c12cafd4f123a2aa0f41691c377a4ee9b6bda880ce2a63c732655640aa124f7e/operationStatuses/08584112417626822610\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6722048/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6722048/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6722048\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 211281f7-1c36-4852-bc41-a0e632529504:9/25/2026 7:39:53 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T19:56:15.641Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T19:56:16.222Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T19:56:16.940Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f8b29484b58f8503a228372e2697e57b73ab90e0e696db73a15cf9aafffa6734 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6722048%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff8b29484b58f8503a228372e2697e57b73ab90e0e696db73a15cf9aafffa6734\ntime=2026-09-25T20:08:40.468Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564524710793216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7165,"pr_state":"open","pr_sha":"4ada4bfca590fd84b0b6be17befd2127f1df09bf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T19:17:19Z","started_at":"2026-09-25T19:17:19Z","completed_at":"2026-09-25T20:19:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:31:33.699151384Z","timing_metadata_checked_at":"2026-09-25T20:31:33.699151384Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:31:33.703902969Z","region_metadata_checked_at":"2026-09-25T20:31:33.703902969Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"173f08b08c90f8bafb7a6e9ec2196fcdaafe7cd10963c214c2dd80ca51d119e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564524710793216","occurred_at":"2026-09-25T19:17:19Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T19:43:33.576Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T19:43:35.122Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T19:43:36.305Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=ab3bdd0fc7529016544686549009b826730fbe35524c4686ce2a5b6ba6eaab9d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0793216%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fab3bdd0fc7529016544686549009b826730fbe35524c4686ce2a5b6ba6eaab9d\ntime=2026-09-25T19:47:37.061Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0793216/providers/Microsoft.Resources/deployments/ab3bdd0fc7529016544686549009b826730fbe35524c4686ce2a5b6ba6eaab9d/operationStatuses/08584112414697532463\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0793216/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0793216/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j0793216\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 929e6247-be1f-43d0-9df0-5b2e7f629d43:9/25/2026 7:44:41 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T19:49:37.061Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T19:49:37.820Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T19:49:38.498Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2d41f8ec48bbeacc0493a4c1faf25af3b1f855750252948156ba8c3d51efbb4b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0793216%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2d41f8ec48bbeacc0493a4c1faf25af3b1f855750252948156ba8c3d51efbb4b\ntime=2026-09-25T20:05:11.187Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0793216/providers/Microsoft.Resources/deployments/2d41f8ec48bbeacc0493a4c1faf25af3b1f855750252948156ba8c3d51efbb4b/operationStatuses/08584112411073941987\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0793216/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0793216/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j0793216\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 5991d836-3e98-49bb-8cbf-7b2517c7e186:9/25/2026 8:04:52 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T20:07:11.191Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:07:11.718Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:07:12.714Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9eec7e7b81fd18e96404172455438d301921cac6d32a893e26e68fab4c4b3a23 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0793216%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9eec7e7b81fd18e96404172455438d301921cac6d32a893e26e68fab4c4b3a23\ntime=2026-09-25T20:13:33.211Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564515059699712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T19:17:17Z","started_at":"2026-09-25T19:17:17Z","completed_at":"2026-09-25T20:16:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:31:33.322462354Z","timing_metadata_checked_at":"2026-09-25T20:31:33.322462354Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:31:33.325287845Z","region_metadata_checked_at":"2026-09-25T20:31:33.325287845Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"738b05341883514088e2583274fd036085f894ff6537200b691c5fdeb125fbea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564515059699712","occurred_at":"2026-09-25T19:17:17Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T19:38:54.486Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T19:38:56.278Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T19:38:57.098Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9075fe028e62a7b71b53fffb0672050d971f8bf5b1fc624c674017c2e444d3e4 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9699712%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9075fe028e62a7b71b53fffb0672050d971f8bf5b1fc624c674017c2e444d3e4\ntime=2026-09-25T19:43:27.741Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9699712/providers/Microsoft.Resources/deployments/9075fe028e62a7b71b53fffb0672050d971f8bf5b1fc624c674017c2e444d3e4/operationStatuses/08584112417489041913\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9699712/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9699712/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9699712\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 49492755-8505-4ffb-bccf-3db2afb9a1e9:9/25/2026 7:39:59 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T19:45:27.742Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T19:45:28.502Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T19:45:29.342Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d29566a890c3012b0c13279e67fa460e95a8eb2242a6c1af4ef4e9f385af0021 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9699712%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd29566a890c3012b0c13279e67fa460e95a8eb2242a6c1af4ef4e9f385af0021\ntime=2026-09-25T20:01:01.444Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9699712/providers/Microsoft.Resources/deployments/d29566a890c3012b0c13279e67fa460e95a8eb2242a6c1af4ef4e9f385af0021/operationStatuses/08584112413566568905\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9699712/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9699712/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9699712\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report a580ef1e-5f91-4209-b107-351c7c388ae8:9/25/2026 8:00:37 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T20:03:01.445Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T20:03:02.537Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T20:03:03.483Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=dc0f2e25c23b9aba88b769d5606728de35ee6589a5d5bc1234b0371122dbda38 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9699712%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fdc0f2e25c23b9aba88b769d5606728de35ee6589a5d5bc1234b0371122dbda38\ntime=2026-09-25T20:08:54.219Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564467513069568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7176,"pr_state":"closed","pr_sha":"a90ad015f6e70d63790d33a15a5294c05dbf614b","final_merged_sha":"a90ad015f6e70d63790d33a15a5294c05dbf614b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T19:17:05Z","started_at":"2026-09-25T19:17:05Z","completed_at":"2026-09-25T21:41:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T21:55:44.543798533Z","timing_metadata_checked_at":"2026-09-25T21:55:44.543798533Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T21:55:44.547000323Z","region_metadata_checked_at":"2026-09-25T21:55:44.547000323Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d6dc7444483938164ef39b40c6355d28d5a5a0a1544e9bc770228c2f1d800c7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564467513069568","occurred_at":"2026-09-25T19:17:05Z","failed_at":"e2e","signature_id":"7e94a80a5cfc3c6e5e5e2149f42307fb2e5227e9fb4287e4913bf9419962a984","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:209]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001afc030\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:209]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001afc030\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103561727021944832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"open","pr_sha":"7b97dd56e1a77f3691c218ccb60e427185cf8340","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T19:06:12Z","started_at":"2026-09-25T19:06:12Z","completed_at":"2026-09-25T19:14:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T19:27:55.716033782Z","timing_metadata_checked_at":"2026-09-25T19:27:55.716033782Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-25T19:27:55.719284072Z","region_metadata_checked_at":"2026-09-25T20:31:33.504722788Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"dab0d4f13fba10e29892fd0a92a048ec88b3c6e6e52d52d2fcf606cf54b77789","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103561727021944832","occurred_at":"2026-09-25T19:06:12Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103558859565043712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7123,"pr_state":"closed","pr_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","final_merged_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T18:54:48Z","started_at":"2026-09-25T18:54:48Z","completed_at":"2026-09-25T21:15:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T21:34:14.26383237Z","timing_metadata_checked_at":"2026-09-25T21:34:14.26383237Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T21:34:14.266670661Z","region_metadata_checked_at":"2026-09-25T21:34:14.266670661Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103556819036147712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"3e077fa473e1d0fb9f27fc5472f089cc2bc994d4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T18:46:42Z","started_at":"2026-09-25T18:46:42Z","completed_at":"2026-09-25T21:03:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T21:24:11.416873355Z","timing_metadata_checked_at":"2026-09-25T21:24:11.416873355Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T21:24:11.420109444Z","region_metadata_checked_at":"2026-09-25T21:24:11.420109444Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"57114f9ca610e4b007e404ee37acaefa1e953cf2bb3b027480c8a3f4f814494e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103556819036147712","occurred_at":"2026-09-25T18:46:42Z","failed_at":"alert","signature_id":"76c4a3536de55d2db8db1324950e8cbfeac28775f3a2d8d76e81d97a1126673d","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T19:55:08Z\n Ended: 2026-09-25T20:04:06Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j6147712-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t27vir57rapi3gvc4mcto7vr7g9fer5-cilium-cluster\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j6147712-mgmt-1\nNamespace: ocm-arohcpci01-2t27vir57rapi3gvc4mcto7vr7g9fer5-cilium-cluster","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"d81c77680ac96c54be3fc098c289cf604df9670fac532335312bcc6bc8680a3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103556819036147712","occurred_at":"2026-09-25T18:46:42Z","failed_at":"alert","signature_id":"a28ad0c15ed158b4ec6df29bb44cfd47d05ec01ee5dc565f62fa878852534253","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T19:58:05Z\n Ended: 2026-09-25T20:31:09Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j6147712-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t27vir57rapi3gvc4mcto7vr7g9fer5-cilium-cluster\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j6147712-mgmt-1\nNamespace: ocm-arohcpci01-2t27vir57rapi3gvc4mcto7vr7g9fer5-cilium-cluster","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103555692374790144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7142,"pr_state":"closed","pr_sha":"cfda26d7698257f0afa1b25f95e39ab26e8f37ef","final_merged_sha":"cfda26d7698257f0afa1b25f95e39ab26e8f37ef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T18:42:13Z","started_at":"2026-09-25T18:42:13Z","completed_at":"2026-09-25T21:03:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T21:24:11.518880433Z","timing_metadata_checked_at":"2026-09-25T21:24:11.518880433Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T21:24:11.522459322Z","region_metadata_checked_at":"2026-09-25T21:24:11.522459322Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1dddab6d59b52ecfcb65e8b39f236a2d1cedcdb33f4a2f25f50fb673c4e3ad60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103555692374790144","occurred_at":"2026-09-25T18:42:13Z","failed_at":"e2e","signature_id":"6875b386161e5cce22e20953ae6d6d2de2afa405a8d3aec98377140ad121d02b","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002d20140\u003e: \n DNS for route host agnhost-e2e-sample-app-rfv4s.apps.aro.v20260901.pw4y.j4790144.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rfv4s.apps.aro.v20260901.pw4y.j4790144.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rfv4s.apps.aro.v20260901.pw4y.j4790144.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002d20140\u003e: \n DNS for route host agnhost-e2e-sample-app-rfv4s.apps.aro.v20260901.pw4y.j4790144.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rfv4s.apps.aro.v20260901.pw4y.j4790144.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rfv4s.apps.aro.v20260901.pw4y.j4790144.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103551058646142976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7118,"pr_state":"closed","pr_sha":"f23942d30dc47ec7edeebb24ec30f183a59ea2fd","final_merged_sha":"f23942d30dc47ec7edeebb24ec30f183a59ea2fd","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T18:23:49Z","started_at":"2026-09-25T18:23:49Z","completed_at":"2026-09-25T20:37:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:52:34.151500514Z","timing_metadata_checked_at":"2026-09-25T20:52:34.151500514Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:52:34.155478104Z","region_metadata_checked_at":"2026-09-25T20:52:34.155478104Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103549055647879168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7168,"pr_state":"closed","pr_sha":"438a454514812b6dc5eb79970e7dce00ce113844","final_merged_sha":"438a454514812b6dc5eb79970e7dce00ce113844","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T18:15:51Z","started_at":"2026-09-25T18:15:51Z","completed_at":"2026-09-25T20:30:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:41:50.724683444Z","timing_metadata_checked_at":"2026-09-25T20:41:50.724683444Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:41:50.728491131Z","region_metadata_checked_at":"2026-09-25T20:41:50.728491131Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"289440340647e5e299696ee8a47121ef54966cc87de28216efb2e2ab836a4e39","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103549055647879168","occurred_at":"2026-09-25T18:15:51Z","failed_at":"e2e","signature_id":"c3b9b0160dc46937aaa2f233272eae5b30c1b9f32ad52223e551b6235fac54c8","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d12c80\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-tvqthjct49m6\" to finish creating: GET https://rp.j7879168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d574528-4ed3-4914-89a8-24518c1433b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d574528-4ed3-4914-89a8-24518c1433b8\",\n \"name\": \"5d574528-4ed3-4914-89a8-24518c1433b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T19:04:01.460224052Z\",\n \"endTime\": \"2026-09-25T19:33:07.221268822Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d12c80\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-tvqthjct49m6\" to finish creating: GET https://rp.j7879168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d574528-4ed3-4914-89a8-24518c1433b8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d574528-4ed3-4914-89a8-24518c1433b8\",\n \"name\": \"5d574528-4ed3-4914-89a8-24518c1433b8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T19:04:01.460224052Z\",\n \"endTime\": \"2026-09-25T19:33:07.221268822Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"638ff47104d392524f473889266fdd83eafc888fceff16831ff392acd25e8736","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103549055647879168","occurred_at":"2026-09-25T18:15:51Z","failed_at":"alert","signature_id":"86cb6bb93c82d8325bd82f0286724ad121ef3f1b1d15c8476cc5376871629ce1","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T19:39:56Z\n Ended: 2026-09-25T19:45:57Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7879168-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7879168-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"b6a7de681a4c13014722a4154033ae667665514a6503b7ddc9e95b0cf35318ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103549055647879168","occurred_at":"2026-09-25T18:15:51Z","failed_at":"alert","signature_id":"d3f2c315f411e3fb04d632b66f8a302db0d803f5f870491f66c91c0c633ac840","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T19:26:00Z\n Ended: 2026-09-25T19:56:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7879168-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7879168/providers/microsoft.monitor/accounts/hcps-j7879168\", namespace=\"ocm-arohcpci01-2t27f8egsdoeqjvu0j3vd58dlko2an9h-r5q1e3i5h8g3j3f\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"fd1ced12-4634-450b-be94-fe821eb628d0\"\n Description: Pod ocm-arohcpci01-2t27f8egsdoeqjvu0j3vd58dlko2an9h-r5q1e3i5h8g3j3f/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-25T19:26:00Z\n Ended: 2026-09-25T19:55:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7879168-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7879168/providers/microsoft.monitor/accounts/hcps-j7879168\", namespace=\"ocm-arohcpci01-2t27f8egsdoeqjvu0j3vd58dlko2an9h-r5q1e3i5h8g3j3f\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"fd1ced12-4634-450b-be94-fe821eb628d0\"\n Description: Pod ocm-arohcpci01-2t27f8egsdoeqjvu0j3vd58dlko2an9h-r5q1e3i5h8g3j3f/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"dda63b275745ba499b3609b3420ed25298aa33ba5d30db3b4b9675293a71e06b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103549055647879168","occurred_at":"2026-09-25T18:15:51Z","failed_at":"alert","signature_id":"36613c51dc94f1ebf66f56c186a77e9398994f4128abf41aa5fe8abb52c4e2a8","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T19:24:04Z\n Ended: 2026-09-25T19:55:01Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7879168-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7879168/providers/microsoft.monitor/accounts/hcps-j7879168\", namespace=\"ocm-arohcpci01-2t27f8egsdoeqjvu0j3vd58dlko2an9h-r5q1e3i5h8g3j3f\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-ftvv6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t27f8egsdoeqjvu0j3vd58dlko2an9h-r5q1e3i5h8g3j3f/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-25T19:24:04Z\n Ended: 2026-09-25T19:40:02Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7879168-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7879168/providers/microsoft.monitor/accounts/hcps-j7879168\", namespace=\"ocm-arohcpci01-2t27f8egsdoeqjvu0j3vd58dlko2an9h-r5q1e3i5h8g3j3f\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-ftvv6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t27f8egsdoeqjvu0j3vd58dlko2an9h-r5q1e3i5h8g3j3f/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"e67732757b709f6ddbad452c472463b73ac1f30adc35f5073802974813be7e92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103549055647879168","occurred_at":"2026-09-25T18:15:51Z","failed_at":"alert","signature_id":"610d97b5c38aad5358d95f8441cb315dd1aae476119a3ff7e9090737558bd925","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T19:26:52Z\n Ended: 2026-09-25T19:33:50Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j7879168-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t27f39puke0sb3qe61e8d6ncedrr4fe-s7p5n9n6h7p3b7d\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j7879168-mgmt-1\nNamespace: ocm-arohcpci01-2t27f39puke0sb3qe61e8d6ncedrr4fe-s7p5n9n6h7p3b7d","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103547560068124672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7164,"pr_state":"closed","pr_sha":"653ff4451ef4e6d7031565b7f3e25b8a6be9a485","final_merged_sha":"653ff4451ef4e6d7031565b7f3e25b8a6be9a485","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T18:09:53Z","started_at":"2026-09-25T18:09:53Z","completed_at":"2026-09-25T20:20:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:31:33.5971971Z","timing_metadata_checked_at":"2026-09-25T20:31:33.5971971Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:31:33.599953392Z","region_metadata_checked_at":"2026-09-25T20:31:33.599953392Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"bc736ce9d795b337ad840648d7181ca1979b62a78172bf95ae20bb5826a14fb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103547560068124672","occurred_at":"2026-09-25T18:09:53Z","failed_at":"alert","signature_id":"07b639689eadc8a3b695373d070cc3ec10682dbbfd11436fc6587f411d82545e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T19:33:39Z\n Ended: 2026-09-25T19:39:40Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8124672-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8124672-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"e32cd917031d66106ac1ad852579defdd0034395d749ca92f4ebd8ba2a0e33dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103547560068124672","occurred_at":"2026-09-25T18:09:53Z","failed_at":"e2e","signature_id":"0911bfb451cecb71d902c4d9a03521c3a5e579e6640a3eb871d2b06f1fb8defa","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:137]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7d980\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-zlq2sfq4grml\" to finish creating: GET https://rp.j8124672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f0ffedb5-9117-42d3-a6d0-4533097f5580\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f0ffedb5-9117-42d3-a6d0-4533097f5580\",\n \"name\": \"f0ffedb5-9117-42d3-a6d0-4533097f5580\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T19:12:52.186530809Z\",\n \"endTime\": \"2026-09-25T19:31:54.854421453Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-bfqvj-mnbtw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-zlq2sfq4grml--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-bfqvj-mnbtw to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-bfqvj-mnbtw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-zlq2sfq4grml--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-bfqvj-mnbtw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:137]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7d980\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-zlq2sfq4grml\" to finish creating: GET https://rp.j8124672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f0ffedb5-9117-42d3-a6d0-4533097f5580\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f0ffedb5-9117-42d3-a6d0-4533097f5580\",\n \"name\": \"f0ffedb5-9117-42d3-a6d0-4533097f5580\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T19:12:52.186530809Z\",\n \"endTime\": \"2026-09-25T19:31:54.854421453Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-bfqvj-mnbtw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-zlq2sfq4grml--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-bfqvj-mnbtw to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-bfqvj-mnbtw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-zlq2sfq4grml--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-bfqvj-mnbtw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103547336255868928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"929dedd75f499475700cf48a143857252d692e54","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T18:09:01Z","started_at":"2026-09-25T18:09:01Z","completed_at":"2026-09-25T20:22:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:41:50.89268936Z","timing_metadata_checked_at":"2026-09-25T20:41:50.89268936Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:41:50.89558595Z","region_metadata_checked_at":"2026-09-25T20:41:50.89558595Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0e79e3a2daf5b8cce306d026232ff35d24b996f04a13e4072989400ea13f3cc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103547336255868928","occurred_at":"2026-09-25T18:09:01Z","failed_at":"e2e","signature_id":"06d4cbae5cb178dd7474a8e93fa31108efbbc06d8bf341e39e73e3837d942123","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1a600\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-sm4jtnxclsxs\" to finish creating: GET https://rp.j5868928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d34f713-a16b-433b-99d2-1c7b278e6b21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d34f713-a16b-433b-99d2-1c7b278e6b21\",\n \"name\": \"6d34f713-a16b-433b-99d2-1c7b278e6b21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T19:01:13.53147986Z\",\n \"endTime\": \"2026-09-25T19:30:14.113086545Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 3m12s ago; version 4.21.35 is Partial (want Completed), started 4m17s ago; version 4.21.35 is Partial (want Completed), started 5m48s ago; version 4.21.35 is Partial (want Completed), started 13m55s ago; version 4.21.35 is Partial (want Completed), started 15m34s ago; version 4.21.35 is Partial (want Completed), started 19m39s ago; version 4.21.35 is Partial (want Completed), started 23m46s ago; version 4.21.35 is Partial (want Completed), started 24m59s ago; version 4.21.35 is Partial (want Completed), started 25m33s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1a600\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-sm4jtnxclsxs\" to finish creating: GET https://rp.j5868928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d34f713-a16b-433b-99d2-1c7b278e6b21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d34f713-a16b-433b-99d2-1c7b278e6b21\",\n \"name\": \"6d34f713-a16b-433b-99d2-1c7b278e6b21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T19:01:13.53147986Z\",\n \"endTime\": \"2026-09-25T19:30:14.113086545Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 3m12s ago; version 4.21.35 is Partial (want Completed), started 4m17s ago; version 4.21.35 is Partial (want Completed), started 5m48s ago; version 4.21.35 is Partial (want Completed), started 13m55s ago; version 4.21.35 is Partial (want Completed), started 15m34s ago; version 4.21.35 is Partial (want Completed), started 19m39s ago; version 4.21.35 is Partial (want Completed), started 23m46s ago; version 4.21.35 is Partial (want Completed), started 24m59s ago; version 4.21.35 is Partial (want Completed), started 25m33s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"50d71e86102fe885def0430fc65ae5eda7365e37815a984b65ed42e4e299b068","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"729a3b9ec3568b28fa39df0cff1337c049111e2e336e8dae30b41be87b002aac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103547336255868928","occurred_at":"2026-09-25T18:09:01Z","failed_at":"e2e","signature_id":"5673b7d1af626ca3af29742bda73744e5ecefedca976779ee4bbf1bbdd725701","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:334]: failed to pull image from private ACR acrpullcreatebgsbfthf8tqk.azurecr.io — the credential provider did not authenticate the pull\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001270ec0\u003e: \n VerifyImagePulled(acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal) timed out after 3m0.007s: expected image from acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:\n - pod acr-pull-test container acr-pull-test: ImagePullBackOff - Back-off pulling image \"acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal:latest: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 08125a7d-2aeb-4cda-8888-4d2636d60fd2; artifact err: get manifest: build image source: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 48c612c6-0cab-4944-9d49-fbdaec01dcc2\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:334]: failed to pull image from private ACR acrpullcreatebgsbfthf8tqk.azurecr.io — the credential provider did not authenticate the pull\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001270ec0\u003e: \n VerifyImagePulled(acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal) timed out after 3m0.007s: expected image from acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:\n - pod acr-pull-test container acr-pull-test: ImagePullBackOff - Back-off pulling image \"acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal:latest: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 08125a7d-2aeb-4cda-8888-4d2636d60fd2; artifact err: get manifest: build image source: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 48c612c6-0cab-4944-9d49-fbdaec01dcc2\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"278b745a86bd068f5605c35799bb8d30b899c82f446eb91deb2f0cab51e1e047","failure_pattern":"VerifyImagePulled(acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal) timed out after 3m0.007s: expected image from acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:","search_query":"VerifyImagePulled(acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal) timed out after 3m0.007s: expected image from acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["278b745a86bd068f5605c35799bb8d30b899c82f446eb91deb2f0cab51e1e047","50d71e86102fe885def0430fc65ae5eda7365e37815a984b65ed42e4e299b068"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103545360231174144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7146,"pr_state":"closed","pr_sha":"34a27988a2da6ca8b624fbab48af075e636b7eb6","final_merged_sha":"34a27988a2da6ca8b624fbab48af075e636b7eb6","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T18:01:05Z","started_at":"2026-09-25T18:01:05Z","completed_at":"2026-09-25T18:51:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T19:07:10.021950441Z","timing_metadata_checked_at":"2026-09-25T19:07:10.021950441Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-25T19:07:10.025748229Z","region_metadata_checked_at":"2026-09-25T20:09:46.282271363Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e354ff286fb0660e90baedabe537dd39eca8a9ae773873e41fa651c50aab78c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103545360231174144","occurred_at":"2026-09-25T18:01:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103540682709798912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6968,"pr_state":"open","pr_sha":"01f9f1a3abc4170f539887eb1b492dccdcdac1e6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-25T17:42:35Z","started_at":"2026-09-25T17:42:35Z","completed_at":"2026-09-25T20:04:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:20:40.961931337Z","timing_metadata_checked_at":"2026-09-25T20:20:40.961931337Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:20:40.966563922Z","region_metadata_checked_at":"2026-09-25T20:20:40.966563922Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103534322014228480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7157,"pr_state":"closed","pr_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","final_merged_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T17:17:18Z","started_at":"2026-09-25T17:17:18Z","completed_at":"2026-09-25T19:31:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T19:49:10.272112952Z","timing_metadata_checked_at":"2026-09-25T19:49:10.272112952Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T19:49:10.274728343Z","region_metadata_checked_at":"2026-09-25T19:49:10.274728343Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3565ddddab00fe44737096689fb0e977a16398a89959c5757eb858102de5388c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103534322014228480","occurred_at":"2026-09-25T17:17:18Z","failed_at":"e2e","signature_id":"db842ed64a05de755e22b9021ea7593d77cdd91048ac8b40b56468fc1ec8705d","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00150e5d0\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-8x29dg6bs8tb\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-8x29dg6bs8tb, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00150e5d0\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-8x29dg6bs8tb\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-8x29dg6bs8tb, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-w8sl5z8nrnnt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-w8sl5z8nrnnt, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103526925438816256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7170,"pr_state":"closed","pr_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","final_merged_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T16:47:55Z","started_at":"2026-09-25T16:47:55Z","completed_at":"2026-09-25T19:04:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T19:17:17.856267914Z","timing_metadata_checked_at":"2026-09-25T19:17:17.856267914Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T19:17:17.859560904Z","region_metadata_checked_at":"2026-09-25T19:17:17.859560904Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"32dd6e0cab78debb857445635a30fa19f74472acf8b5bdcdd287e4b8f0200aed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103526925438816256","occurred_at":"2026-09-25T16:47:55Z","failed_at":"alert","signature_id":"17ad0b9bb6164e3d6001f2d6865aa89ffb225847638c9542425a50a55f53006d","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T18:02:18Z\n Ended: 2026-09-25T18:09:19Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j8816256-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t2671elvg1k4fc73r11qe4v243n9j7f-v20260901\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j8816256-mgmt-2\nNamespace: ocm-arohcpci01-2t2671elvg1k4fc73r11qe4v243n9j7f-v20260901\n\nFiring 2:\n State: Resolved\n Started: 2026-09-25T18:18:19Z\n Ended: 2026-09-25T18:24:18Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j8816256-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t2670enml25if5q1da1ra9v7u3tj5tb-q6q2g5a5a3y2p8o\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j8816256-mgmt-1\nNamespace: ocm-arohcpci01-2t2670enml25if5q1da1ra9v7u3tj5tb-q6q2g5a5a3y2p8o","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}},{"row_id":"d34d680d33250019ffea9ebe92d3fed9e6b4df31fa5092bd9fa299fecabad974","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103526925438816256","occurred_at":"2026-09-25T16:47:55Z","failed_at":"alert","signature_id":"e8f5b6bb718794f5ad75fa1fc52fd2b7f6cdb8275ea9e32cd99aac68f99b15e3","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T18:02:18Z\n Ended: 2026-09-25T18:08:18Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j8816256-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t26709mib68aq55h01quvi900cl26b5-j0a4i6b0g6m6j8j\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j8816256-mgmt-1\nNamespace: ocm-arohcpci01-2t26709mib68aq55h01quvi900cl26b5-j0a4i6b0g6m6j8j","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"fcb6f5eab59be92c204594a293145f6710f1926db60c8d678d7d182d10d85ae8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103526925438816256","occurred_at":"2026-09-25T16:47:55Z","failed_at":"alert","signature_id":"74115cd9a87272e438969badac47448ecb69890d765eba6086cb2e71bf1e9e4e","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T18:05:20Z\n Ended: 2026-09-25T18:34:18Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j8816256-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t2671elvg1k4fc73r11qe4v243n9j7f-v20260901\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j8816256-mgmt-2\nNamespace: ocm-arohcpci01-2t2671elvg1k4fc73r11qe4v243n9j7f-v20260901","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103523744113758208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7174,"pr_state":"closed","pr_sha":"6622cab9ad4d278b9626d29a58f5bbe3a9f5fdc9","final_merged_sha":"756dbd0aea05ea7d18d18b94d48e2ccdb7df1c9c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T16:35:16Z","started_at":"2026-09-25T16:35:16Z","completed_at":"2026-09-25T18:53:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T19:07:10.232878475Z","timing_metadata_checked_at":"2026-09-25T19:07:10.232878475Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T19:07:10.235601167Z","region_metadata_checked_at":"2026-09-25T19:07:10.235601167Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"591586a79aaa535cd4fb9f5829b44af4e5cd798dac98bb8b5b8a1375fd67adec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103523744113758208","occurred_at":"2026-09-25T16:35:16Z","failed_at":"alert","signature_id":"9e87fb340216b923c3b3e1c2c19a207c8d191371d5639a1834a3d1a4fa9ac16f","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:47:38Z\n Ended: 2026-09-25T18:15:38Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3758208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.74:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3758208/providers/microsoft.monitor/accounts/hcps-j3758208\", namespace=\"ocm-arohcpci01-2t262eoagocot786h611d0fvg0ggb7j6-acr-pull-create\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-6vl2d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t262eoagocot786h611d0fvg0ggb7j6-acr-pull-create/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-25T17:57:37Z\n Ended: 2026-09-25T18:19:42Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3758208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.74:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3758208/providers/microsoft.monitor/accounts/hcps-j3758208\", namespace=\"ocm-arohcpci01-2t262eoagocot786h611d0fvg0ggb7j6-acr-pull-create\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-6vl2d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t262eoagocot786h611d0fvg0ggb7j6-acr-pull-create/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"d4cc9c2481829a5d93e879233cbf2228c391f577c77a058d48f7c380117bd794","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103523744113758208","occurred_at":"2026-09-25T16:35:16Z","failed_at":"alert","signature_id":"8d79d2e209e5007a23d684b8f03e30c1b2188d6188648ba6feeec72bedbded21","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T18:02:42Z\n Ended: 2026-09-25T18:15:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3758208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.74:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3758208/providers/microsoft.monitor/accounts/hcps-j3758208\", namespace=\"ocm-arohcpci01-2t262eoagocot786h611d0fvg0ggb7j6-acr-pull-create\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-6vl2d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t262eoagocot786h611d0fvg0ggb7j6-acr-pull-create/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"fb5c6468a2d53cd8f984bf6368ff560462f5486089922010f4f8114d3deebf14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103523744113758208","occurred_at":"2026-09-25T16:35:16Z","failed_at":"e2e","signature_id":"837f249c37264e4b948db37729027e6d876d161f3cbaadf548aa629a76406376","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d874a0\u003e: \n failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-vdsj46l8n4nc\" to finish creating: GET https://rp.j3758208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc17ba24-1832-41fc-ac72-c4255c121a55\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc17ba24-1832-41fc-ac72-c4255c121a55\",\n \"name\": \"bc17ba24-1832-41fc-ac72-c4255c121a55\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T17:28:26.003020977Z\",\n \"endTime\": \"2026-09-25T17:57:27.213791219Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d874a0\u003e: \n failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-vdsj46l8n4nc\" to finish creating: GET https://rp.j3758208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc17ba24-1832-41fc-ac72-c4255c121a55\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc17ba24-1832-41fc-ac72-c4255c121a55\",\n \"name\": \"bc17ba24-1832-41fc-ac72-c4255c121a55\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T17:28:26.003020977Z\",\n \"endTime\": \"2026-09-25T17:57:27.213791219Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103520651284320256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7104,"pr_state":"closed","pr_sha":"ebc2d5230a6dd32349f721bbfe1d38e63e8de0ef","final_merged_sha":"a2e35daa20b37841bf1d2f73e88d95554cdc5664","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T16:22:59Z","started_at":"2026-09-25T16:22:59Z","completed_at":"2026-09-25T16:56:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T17:13:32.42391642Z","timing_metadata_checked_at":"2026-09-25T17:13:32.42391642Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-09-25T17:13:32.429135004Z","region_metadata_checked_at":"2026-09-25T18:15:14.363729679Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"cf7955d7fe1957cd99116cff8fa155f86a42d9056af8e602f88b20cfea443644","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103520651284320256","occurred_at":"2026-09-25T16:22:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519763601821696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7165,"pr_state":"open","pr_sha":"4ada4bfca590fd84b0b6be17befd2127f1df09bf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T16:19:27Z","started_at":"2026-09-25T16:19:27Z","completed_at":"2026-09-25T18:47:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T18:56:59.430646794Z","timing_metadata_checked_at":"2026-09-25T18:56:59.430646794Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T18:56:59.434024684Z","region_metadata_checked_at":"2026-09-25T18:56:59.434024684Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"14e636923e3a59a6256b2a29d443c7e49a2fb51c84a92d32d686f8176c8dd11a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519763601821696","occurred_at":"2026-09-25T16:19:27Z","failed_at":"alert","signature_id":"d4ed3df415fbe310f0efa530278cd9d07d7e76df5d066f5aaf8a375b9e6f0c9f","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:37:49Z\n Ended: 2026-09-25T18:09:50Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j1821696-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t260b63vr30ansdufkh8dg5iud39pap-v3v9a3e5h3r8h7v\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j1821696-mgmt-1\nNamespace: ocm-arohcpci01-2t260b63vr30ansdufkh8dg5iud39pap-v3v9a3e5h3r8h7v","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"230ac7ec645dee34b7777979a06f9f2d0b72c1ae2b01dafd382cd5254d586157","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519763601821696","occurred_at":"2026-09-25T16:19:27Z","failed_at":"alert","signature_id":"bfe979a56396aa9f080f5376ebac69b7f4a63495cb93eab7c4de33390c9d9d3f","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:30:49Z\n Ended: 2026-09-25T17:44:53Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j1821696-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t260b63vr30ansdufkh8dg5iud39pap-v3v9a3e5h3r8h7v\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j1821696-mgmt-1\nNamespace: ocm-arohcpci01-2t260b63vr30ansdufkh8dg5iud39pap-v3v9a3e5h3r8h7v","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"fa1b394a0bec0edb7db9d1b14e5b63aee9f256497a470c6fdb6563872c47fb2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519763601821696","occurred_at":"2026-09-25T16:19:27Z","failed_at":"alert","signature_id":"6072cc6e524eb8561dccfb6e08abf762fc8d80b75bf6be872960a1df3249592d","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:55:16Z\n Ended: 2026-09-25T18:03:15Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1821696-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1821696-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519758426050560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"54fa83b6be2bda741a3e467491a8b8caf8a2dbe3","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T16:19:26Z","started_at":"2026-09-25T16:19:26Z","completed_at":"2026-09-25T18:46:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T18:56:59.330456107Z","timing_metadata_checked_at":"2026-09-25T18:56:59.330456107Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T18:56:59.334487095Z","region_metadata_checked_at":"2026-09-25T18:56:59.334487095Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"07a96bb0393dfe9ad7aa3551280f83c3357c774f2b7035d03e29c61905b0c25b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519758426050560","occurred_at":"2026-09-25T16:19:26Z","failed_at":"alert","signature_id":"111b6f7f301fb2f1efa95a5ddf7cc001a54bed1e2f2b096078dd112e0cbf027b","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:36:24Z\n Ended: 2026-09-25T17:54:25Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j6050560-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t260i936l6vnaoo8r9a7o3fqfkl3jcc-f6f8f6c5n1j9d4v\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j6050560-mgmt-1\nNamespace: ocm-arohcpci01-2t260i936l6vnaoo8r9a7o3fqfkl3jcc-f6f8f6c5n1j9d4v","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}},{"row_id":"2401e6cdb6821c2c91d3d34c342d920e35690f0fcc98ebdc6e7b0831956f16ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519758426050560","occurred_at":"2026-09-25T16:19:26Z","failed_at":"alert","signature_id":"301e2cf0d90497c804a3b815b89a2f5d8dbae118206c74ca0baaf6b49f6f0d9b","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:39:22Z\n Ended: 2026-09-25T18:20:24Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j6050560-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t260i936l6vnaoo8r9a7o3fqfkl3jcc-f6f8f6c5n1j9d4v\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j6050560-mgmt-1\nNamespace: ocm-arohcpci01-2t260i936l6vnaoo8r9a7o3fqfkl3jcc-f6f8f6c5n1j9d4v","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"4aa43eca2a7d6099d7d82701be2a7f05acabbcb1c33c81a480e958abd51c7dc8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519758426050560","occurred_at":"2026-09-25T16:19:26Z","failed_at":"alert","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:47:24Z\n Ended: 2026-09-25T17:56:25Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6050560-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"9ca7589bb0daa8c75deb93ec1d10167f64827484695bb6d7f6e7256628474a85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519758426050560","occurred_at":"2026-09-25T16:19:26Z","failed_at":"e2e","signature_id":"5cd60a99f0ee5798d016bc027d90478a36bcd3fdf3a9be5ae03b31e9803d6e8e","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc003af2080\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dnr4nbhwmf28\" to finish creating: GET https://rp.j6050560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/845fcbd5-9444-4826-9970-7a1c7611c2b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/845fcbd5-9444-4826-9970-7a1c7611c2b7\",\n \"name\": \"845fcbd5-9444-4826-9970-7a1c7611c2b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T17:37:30.398464058Z\",\n \"endTime\": \"2026-09-25T17:56:37.126932559Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e4o0a9k9h1t4n3s-np-deletable-7xqlr-j7hs9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dnr4nbhwmf28--managed/providers/Microsoft.Compute/virtualMachines/e4o0a9k9h1t4n3s-np-deletable-7xqlr-j7hs9 to exist\\nMachine e4o0a9k9h1t4n3s-np-deletable-7xqlr-shwl7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dnr4nbhwmf28--managed/providers/Microsoft.Compute/virtualMachines/e4o0a9k9h1t4n3s-np-deletable-7xqlr-shwl7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e4o0a9k9h1t4n3s-np-deletable-7xqlr-j7hs9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dnr4nbhwmf28--managed/providers/Microsoft.Compute/virtualMachines/e4o0a9k9h1t4n3s-np-deletable-7xqlr-j7hs9 to exist\\nMachine e4o0a9k9h1t4n3s-np-deletable-7xqlr-shwl7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dnr4nbhwmf28--managed/providers/Microsoft.Compute/virtualMachines/e4o0a9k9h1t4n3s-np-deletable-7xqlr-shwl7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc003af2080\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dnr4nbhwmf28\" to finish creating: GET https://rp.j6050560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/845fcbd5-9444-4826-9970-7a1c7611c2b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/845fcbd5-9444-4826-9970-7a1c7611c2b7\",\n \"name\": \"845fcbd5-9444-4826-9970-7a1c7611c2b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T17:37:30.398464058Z\",\n \"endTime\": \"2026-09-25T17:56:37.126932559Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e4o0a9k9h1t4n3s-np-deletable-7xqlr-j7hs9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dnr4nbhwmf28--managed/providers/Microsoft.Compute/virtualMachines/e4o0a9k9h1t4n3s-np-deletable-7xqlr-j7hs9 to exist\\nMachine e4o0a9k9h1t4n3s-np-deletable-7xqlr-shwl7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dnr4nbhwmf28--managed/providers/Microsoft.Compute/virtualMachines/e4o0a9k9h1t4n3s-np-deletable-7xqlr-shwl7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e4o0a9k9h1t4n3s-np-deletable-7xqlr-j7hs9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dnr4nbhwmf28--managed/providers/Microsoft.Compute/virtualMachines/e4o0a9k9h1t4n3s-np-deletable-7xqlr-j7hs9 to exist\\nMachine e4o0a9k9h1t4n3s-np-deletable-7xqlr-shwl7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dnr4nbhwmf28--managed/providers/Microsoft.Compute/virtualMachines/e4o0a9k9h1t4n3s-np-deletable-7xqlr-shwl7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dnr4nbhwmf28\" to finish creating: GET https://rp"}},{"row_id":"b12ab06b8c67206fa2908d8b741ea1d78dd85b588613a9f770edf9b72deb6d02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519758426050560","occurred_at":"2026-09-25T16:19:26Z","failed_at":"alert","signature_id":"bd72e06f6355865d4197eaf172fe11459b96dc4ec574f9d4b405ece644f53733","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T18:00:13Z\n Ended: 2026-09-25T18:06:11Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6050560-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6050560-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519728415805440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T16:19:18Z","started_at":"2026-09-25T16:19:18Z","completed_at":"2026-09-25T18:56:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T19:07:09.802037536Z","timing_metadata_checked_at":"2026-09-25T19:07:09.802037536Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T19:07:09.805042326Z","region_metadata_checked_at":"2026-09-25T19:07:09.805042326Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"4e82ea084d0b4d285cd735a40cd7a90c5432f8d9659ed8308b03f6574bb3740c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519728415805440","occurred_at":"2026-09-25T16:19:18Z","failed_at":"alert","signature_id":"e3c1944ee456bdc02cd730f513d15fdcf94aa9db83c39ed1a308fd06ddef0926","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:59:16Z\n Ended: 2026-09-25T18:05:17Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5805440-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5805440-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519715358937088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T16:19:16Z","started_at":"2026-09-25T16:19:16Z","completed_at":"2026-09-25T18:50:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T19:07:09.9082966Z","timing_metadata_checked_at":"2026-09-25T19:07:09.9082966Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T19:07:09.911340491Z","region_metadata_checked_at":"2026-09-25T19:07:09.911340491Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2d4dd1562f9c18f0f0f8c218d49c6185734c436075cc476704c56e890c7f0cdd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519715358937088","occurred_at":"2026-09-25T16:19:16Z","failed_at":"e2e","signature_id":"80d23cd3f5a37fa8cc3ba788b04667c12b2e9abb864a0b69c2b97bc1936b5e91","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4e300\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-k7vshcl5nh9s\" to finish creating: GET https://rp.j8937088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/60dc2458-30c4-43c0-9c09-17e7a502b57d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/60dc2458-30c4-43c0-9c09-17e7a502b57d\",\n \"name\": \"60dc2458-30c4-43c0-9c09-17e7a502b57d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T17:21:27.919326113Z\",\n \"endTime\": \"2026-09-25T17:50:31.48468713Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 35s ago; version 4.21.35 is Partial (want Completed), started 3m12s ago; version 4.21.35 is Partial (want Completed), started 5m19s ago; version 4.21.35 is Partial (want Completed), started 6m43s ago; version 4.21.35 is Partial (want Completed), started 8m41s ago; version 4.21.35 is Partial (want Completed), started 12m14s ago; version 4.21.35 is Partial (want Completed), started 13m25s ago; version 4.21.35 is Partial (want Completed), started 13m48s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4e300\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-k7vshcl5nh9s\" to finish creating: GET https://rp.j8937088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/60dc2458-30c4-43c0-9c09-17e7a502b57d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/60dc2458-30c4-43c0-9c09-17e7a502b57d\",\n \"name\": \"60dc2458-30c4-43c0-9c09-17e7a502b57d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T17:21:27.919326113Z\",\n \"endTime\": \"2026-09-25T17:50:31.48468713Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 35s ago; version 4.21.35 is Partial (want Completed), started 3m12s ago; version 4.21.35 is Partial (want Completed), started 5m19s ago; version 4.21.35 is Partial (want Completed), started 6m43s ago; version 4.21.35 is Partial (want Completed), started 8m41s ago; version 4.21.35 is Partial (want Completed), started 12m14s ago; version 4.21.35 is Partial (want Completed), started 13m25s ago; version 4.21.35 is Partial (want Completed), started 13m48s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9875340340e8a84eab3c14c72cc498c82582aaf6c0fdf1646fe5c4ca338b7548","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f733f3a1b5363356172d18cda8174d8bb01399219ef50d541d21d7bb2dd424da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519715358937088","occurred_at":"2026-09-25T16:19:16Z","failed_at":"alert","signature_id":"4a7342040e61029b69f8b6b07ebb37773b7c8fd7a7d80fc00ca7010aa64d8ffd","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:59:01Z\n Ended: 2026-09-25T18:05:02Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8937088-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8937088-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"f8825d7f9c229d3032491d438351b0ac2fb855e47afe8de311d77dac9764f1ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519715358937088","occurred_at":"2026-09-25T16:19:16Z","failed_at":"e2e","signature_id":"c8f8afa64d574b78b209e2a21b36f1feb7a2662df850c394333f89ba4e19dde5","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c39160\u003e: \n failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-01-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c39160\u003e: \n failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-01-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","failure_pattern":"ERROR CODE: 429; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: 429"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","9875340340e8a84eab3c14c72cc498c82582aaf6c0fdf1646fe5c4ca338b7548","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7163/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519656126976000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7163,"pr_state":"closed","pr_sha":"e1e2c7572fd4860250f0893f8b53c154929b9e83","final_merged_sha":"e1e2c7572fd4860250f0893f8b53c154929b9e83","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T16:18:54Z","started_at":"2026-09-25T16:18:54Z","completed_at":"2026-09-25T18:49:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T19:07:10.117936238Z","timing_metadata_checked_at":"2026-09-25T19:07:10.117936238Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T19:07:10.120398331Z","region_metadata_checked_at":"2026-09-25T19:07:10.120398331Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519651102199808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T16:18:52Z","started_at":"2026-09-25T16:18:52Z","completed_at":"2026-09-25T18:50:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T19:07:09.693796377Z","timing_metadata_checked_at":"2026-09-25T19:07:09.693796377Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T19:07:09.699075761Z","region_metadata_checked_at":"2026-09-25T19:07:09.699075761Z"},"failed_at":["alert"],"failed_test_count":5,"occurrences":[{"row_id":"02a468adf88c3ab4cabd3e2c647c15e085ddf3b51343786c3ae1a394a3cbcd86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519651102199808","occurred_at":"2026-09-25T16:18:52Z","failed_at":"alert","signature_id":"5ff26d2e2494532a8474d2c0bf3bb87491fd5743870688498d16fdf695c61721","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:48:35Z\n Ended: 2026-09-25T18:07:35Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j2199808-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t25tqsa8trspr697l7cko4diuunq4kp-l2z0m1n6e9b8a0d\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2199808-mgmt-1\nNamespace: ocm-arohcpci01-2t25tqsa8trspr697l7cko4diuunq4kp-l2z0m1n6e9b8a0d","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"4ce6358a87353879f38ad25390ce8a98f10723e3ce0dbd8d7c2f7efb551a5525","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519651102199808","occurred_at":"2026-09-25T16:18:52Z","failed_at":"alert","signature_id":"b81daef2683367cbd95fe9df79dfbd725276998f256d289948e3586da1b5177e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:59:08Z\n Ended: 2026-09-25T18:07:12Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2199808-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2199808-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"587157d79a8c62ac0a324ec0b5f62bd732b330fc4bd68112b0f681bfac41bbd2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519651102199808","occurred_at":"2026-09-25T16:18:52Z","failed_at":"alert","signature_id":"2570a67933cbb17e5fa7c5faafbae18b8522372cb31dbe440c9e1e0a8bdc7c05","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:41:35Z\n Ended: 2026-09-25T17:58:35Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j2199808-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t25tqsa8trspr697l7cko4diuunq4kp-l2z0m1n6e9b8a0d\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2199808-mgmt-1\nNamespace: ocm-arohcpci01-2t25tqsa8trspr697l7cko4diuunq4kp-l2z0m1n6e9b8a0d","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}},{"row_id":"ad550a02222dc1f732a44c50d1b9da6179062c43288d3de289560bcb339d7bca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519651102199808","occurred_at":"2026-09-25T16:18:52Z","failed_at":"alert","signature_id":"14f1324b646a515c78bb9b0b3f9684f61e1dd4ba8de64e559e927d9fb3b94c61","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:44:33Z\n Ended: 2026-09-25T17:58:33Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j2199808-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t25tqsa8trspr697l7cko4diuunq4kp-l2z0m1n6e9b8a0d\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2199808-mgmt-1\nNamespace: ocm-arohcpci01-2t25tqsa8trspr697l7cko4diuunq4kp-l2z0m1n6e9b8a0d","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"d63806f5b12d287744282beecc46cd5e63633cf189f8733adfb0b97e37ca41be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519651102199808","occurred_at":"2026-09-25T16:18:52Z","failed_at":"alert","signature_id":"009f41ec16856f6a2f163a6d76e1833343da4f507cba32812733d4138afdbf21","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:44:34Z\n Ended: 2026-09-25T18:24:37Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j2199808-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t25tqsa8trspr697l7cko4diuunq4kp-l2z0m1n6e9b8a0d\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2199808-mgmt-1\nNamespace: ocm-arohcpci01-2t25tqsa8trspr697l7cko4diuunq4kp-l2z0m1n6e9b8a0d","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103518634033811456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"3e077fa473e1d0fb9f27fc5472f089cc2bc994d4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T16:14:55Z","started_at":"2026-09-25T16:14:55Z","completed_at":"2026-09-25T18:34:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T18:46:18.94029898Z","timing_metadata_checked_at":"2026-09-25T18:46:18.94029898Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T18:46:18.943760069Z","region_metadata_checked_at":"2026-09-25T18:46:18.943760069Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"246398502161a80f5a21e0e9e5d94bfab2fda5ba371c6f91eada4305a4b80069","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103518634033811456","occurred_at":"2026-09-25T16:14:55Z","failed_at":"e2e","signature_id":"206eded0e06122e841558aff57990edd60f07f91dcefa176fe7ded75bb991a39","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:444]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0014697a0\u003e: \n VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-5bd8h-kb8d6]\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:444]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0014697a0\u003e: \n VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-5bd8h-kb8d6]\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0f6b13b4a76d7562287bf1face4f5d9fb1b45d4ebf0047290b8a707a05354ee","failure_pattern":"VerifyNodesReady failed: there are not ready nodes: [\u003cnode\u003e]","search_query":"VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-5bd8h-kb8d6]"}},{"row_id":"6aff7bd4d1ba4e98d1ce7d0dcafd557ee7554720c6925fb3d1cf804f5975a6f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103518634033811456","occurred_at":"2026-09-25T16:14:55Z","failed_at":"alert","signature_id":"2a236b2c0e6b67c057608ab27afe1e5a63ddf0d896563f1d1f4e9ce94cf1fbb0","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T17:46:16Z\n Ended: 2026-09-25T17:52:16Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3811456-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3811456-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["b0f6b13b4a76d7562287bf1face4f5d9fb1b45d4ebf0047290b8a707a05354ee","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103507194874957824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"open","pr_sha":"7b97dd56e1a77f3691c218ccb60e427185cf8340","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T15:29:30Z","started_at":"2026-09-25T15:29:30Z","completed_at":"2026-09-25T18:03:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T18:15:14.489937487Z","timing_metadata_checked_at":"2026-09-25T18:15:14.489937487Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T18:15:14.493804575Z","region_metadata_checked_at":"2026-09-25T18:15:14.493804575Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6ce00269a546d1c18bea198a8a9c00dc39bf70d54b1ff8598aab3f36101061a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103507194874957824","occurred_at":"2026-09-25T15:29:30Z","failed_at":"e2e","signature_id":"1890310e193288c8c01c75118137ccf125a485da275b0c15cc82d2f4e04e6b39","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fsqv8x\" for cluster \"cluster-candidate-4-23-fsqv8x\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae0220\u003e: \n failed to create NodePool np-fsqv8x: failed waiting for nodepool=\"np-fsqv8x\" for cluster \"cluster-candidate-4-23-fsqv8x\" in resourcegroup=\"rg-candidate-4-23-pzhj9mqnvlqb\" to finish creating: GET https://rp.j4957824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18751211-9c50-4f3d-bb4a-0903f4b13e1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18751211-9c50-4f3d-bb4a-0903f4b13e1e\",\n \"name\": \"18751211-9c50-4f3d-bb4a-0903f4b13e1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T16:52:45.296063512Z\",\n \"endTime\": \"2026-09-25T17:11:55.439732808Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-4fmpm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-pzhj9mqnvlqb-candidate-fsqv8x--managed/providers/Microsoft.Compute/virtualMachines/p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-4fmpm to exist\\nMachine p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-mhmlq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-pzhj9mqnvlqb-candidate-fsqv8x--managed/providers/Microsoft.Compute/virtualMachines/p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-mhmlq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-4fmpm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-pzhj9mqnvlqb-candidate-fsqv8x--managed/providers/Microsoft.Compute/virtualMachines/p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-4fmpm to exist\\nMachine p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-mhmlq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-pzhj9mqnvlqb-candidate-fsqv8x--managed/providers/Microsoft.Compute/virtualMachines/p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-mhmlq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fsqv8x\" for cluster \"cluster-candidate-4-23-fsqv8x\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae0220\u003e: \n failed to create NodePool np-fsqv8x: failed waiting for nodepool=\"np-fsqv8x\" for cluster \"cluster-candidate-4-23-fsqv8x\" in resourcegroup=\"rg-candidate-4-23-pzhj9mqnvlqb\" to finish creating: GET https://rp.j4957824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18751211-9c50-4f3d-bb4a-0903f4b13e1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18751211-9c50-4f3d-bb4a-0903f4b13e1e\",\n \"name\": \"18751211-9c50-4f3d-bb4a-0903f4b13e1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T16:52:45.296063512Z\",\n \"endTime\": \"2026-09-25T17:11:55.439732808Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-4fmpm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-pzhj9mqnvlqb-candidate-fsqv8x--managed/providers/Microsoft.Compute/virtualMachines/p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-4fmpm to exist\\nMachine p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-mhmlq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-pzhj9mqnvlqb-candidate-fsqv8x--managed/providers/Microsoft.Compute/virtualMachines/p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-mhmlq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-4fmpm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-pzhj9mqnvlqb-candidate-fsqv8x--managed/providers/Microsoft.Compute/virtualMachines/p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-4fmpm to exist\\nMachine p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-mhmlq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-pzhj9mqnvlqb-candidate-fsqv8x--managed/providers/Microsoft.Compute/virtualMachines/p7t0b7i3k0p8m3o-np-fsqv8x-kg66c-mhmlq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"faf942b116205b8a019c49bde9ec275ed5ecce79bf0004f836a65b8372de0f71","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103507194874957824","occurred_at":"2026-09-25T15:29:30Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T16:53:53Z\n Ended: 2026-09-25T17:07:51Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4957824-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103506500650536960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7020,"pr_state":"closed","pr_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","final_merged_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T15:26:45Z","started_at":"2026-09-25T15:26:45Z","completed_at":"2026-09-25T18:12:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T18:25:17.756496211Z","timing_metadata_checked_at":"2026-09-25T18:25:17.756496211Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T18:25:17.759841501Z","region_metadata_checked_at":"2026-09-25T18:25:17.759841501Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103505655406006272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7176,"pr_state":"closed","pr_sha":"a90ad015f6e70d63790d33a15a5294c05dbf614b","final_merged_sha":"a90ad015f6e70d63790d33a15a5294c05dbf614b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T15:23:24Z","started_at":"2026-09-25T15:23:24Z","completed_at":"2026-09-25T18:07:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T18:25:17.884079315Z","timing_metadata_checked_at":"2026-09-25T18:25:17.884079315Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T18:25:17.887103906Z","region_metadata_checked_at":"2026-09-25T18:25:17.887103906Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"501812e374097f22883be2dfd428f300cc09783c9543be0b5c448d8a2176c0e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103505655406006272","occurred_at":"2026-09-25T15:23:24Z","failed_at":"e2e","signature_id":"a9686068b550d21e25c4eca196ba3a0cfd9bb97605a83dcd0d2aca5dd9969175","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:247]: failed to create node pool\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e138c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"acr-pull-create\" in resourcegroup=\"acr-pull-create-l4sqwq4856qf\" to finish creating: GET https://rp.j6006272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d70af1e-4af9-4f19-a168-298a07db118f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d70af1e-4af9-4f19-a168-298a07db118f\",\n \"name\": \"4d70af1e-4af9-4f19-a168-298a07db118f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T16:36:46.813798448Z\",\n \"endTime\": \"2026-09-25T16:55:47.121974819Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine acr-pull-create-np-1-pnchl-ppgqx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-l4sqwq4856qf--managed/providers/Microsoft.Compute/virtualMachines/acr-pull-create-np-1-pnchl-ppgqx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine acr-pull-create-np-1-pnchl-ppgqx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-l4sqwq4856qf--managed/providers/Microsoft.Compute/virtualMachines/acr-pull-create-np-1-pnchl-ppgqx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:247]: failed to create node pool\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e138c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"acr-pull-create\" in resourcegroup=\"acr-pull-create-l4sqwq4856qf\" to finish creating: GET https://rp.j6006272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d70af1e-4af9-4f19-a168-298a07db118f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d70af1e-4af9-4f19-a168-298a07db118f\",\n \"name\": \"4d70af1e-4af9-4f19-a168-298a07db118f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T16:36:46.813798448Z\",\n \"endTime\": \"2026-09-25T16:55:47.121974819Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine acr-pull-create-np-1-pnchl-ppgqx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-l4sqwq4856qf--managed/providers/Microsoft.Compute/virtualMachines/acr-pull-create-np-1-pnchl-ppgqx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine acr-pull-create-np-1-pnchl-ppgqx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-l4sqwq4856qf--managed/providers/Microsoft.Compute/virtualMachines/acr-pull-create-np-1-pnchl-ppgqx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8520e50ab62096e57540f520e4fcaa757d3cdb3c9e4fb313b6520d8b0b8c1b4e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103505655406006272","occurred_at":"2026-09-25T15:23:24Z","failed_at":"e2e","signature_id":"0cdecef159b757ddf931d683bdcc29ffb06fd65143e02ca7f2cf4ee378941f30","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000e94378\u003e: \n resource group \"disabled-image-registry-tvns76rrbqk5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-tvns76rrbqk5\" to finish deleting: GET https://rp.j6006272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73511aff-bb96-4107-9e34-0950b93079eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73511aff-bb96-4107-9e34-0950b93079eb\",\n \"name\": \"73511aff-bb96-4107-9e34-0950b93079eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T16:34:38.978866583Z\",\n \"endTime\": \"2026-09-25T16:58:47.471362642Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t254sr73df6qs4ckqqphfpe2lj40f4b still exists (deletion dispatched at 2026-09-25T16:34:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000e94378\u003e: \n resource group \"disabled-image-registry-tvns76rrbqk5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-tvns76rrbqk5\" to finish deleting: GET https://rp.j6006272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73511aff-bb96-4107-9e34-0950b93079eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73511aff-bb96-4107-9e34-0950b93079eb\",\n \"name\": \"73511aff-bb96-4107-9e34-0950b93079eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T16:34:38.978866583Z\",\n \"endTime\": \"2026-09-25T16:58:47.471362642Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t254sr73df6qs4ckqqphfpe2lj40f4b still exists (deletion dispatched at 2026-09-25T16:34:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"daad111b0ad48b71e15195b2e8cdc7fe975196064f38d16346f0801507e6ace5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103505655406006272","occurred_at":"2026-09-25T15:23:24Z","failed_at":"alert","signature_id":"080e2def46312d4b62996fecf6131bceb6d060ae87b921205929ef11720accf9","test_name":"[aro-hcp-observability] [svc] alert MaestroEventStuckUnreconciled does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T16:32:06Z\n Ended: 2026-09-25T16:39:07Z\n Severity: Sev3\n Labels: alertname=\"MaestroEventStuckUnreconciled\", component=\"maestro\", region=\"westus3\", severity=\"warning\"\n Description: A Maestro event has been unreconciled for 6m 6s. This likely indicates a lost Postgres NOTIFY message. The event will not be processed until the periodic sync runs (default 10 hours).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d95ad1f73f14bcd5a00d745ec172e29456c0e319d8668700b8e6e2c26009682d","failure_pattern":"alert [svc] MaestroEventStuckUnreconciled fired","search_query":"alert [svc] MaestroEventStuckUnreconciled fired"}},{"row_id":"f51768e4d0b3b7de768ee732cd0cae87866026d63ca125fed1d3fc2680ccd5d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103505655406006272","occurred_at":"2026-09-25T15:23:24Z","failed_at":"alert","signature_id":"9c3b815afa64d94bbcc8e738bb6c0fb9a74eada3ce4bf0147c94d3c439b9f138","test_name":"[aro-hcp-observability] [svc] alert MaestroSpecControllerReconcileErrors does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T16:36:07Z\n Ended: 2026-09-25T16:58:05Z\n Severity: Sev3\n Labels: alertname=\"MaestroSpecControllerReconcileErrors\", component=\"maestro\", region=\"westus3\", severity=\"warning\"\n Description: Maestro spec controller reconcile error rate is above 10% for the last 10 minutes. Resources may not be reaching management clusters. Current value: 20.38%.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5121c689bbab23ebd70308f98015ffe29584447a2e07ce08bd8d406a3c32c40f","failure_pattern":"alert [svc] MaestroSpecControllerReconcileErrors fired","search_query":"alert [svc] MaestroSpecControllerReconcileErrors fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","5121c689bbab23ebd70308f98015ffe29584447a2e07ce08bd8d406a3c32c40f","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","d95ad1f73f14bcd5a00d745ec172e29456c0e319d8668700b8e6e2c26009682d"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103505090173210624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T15:21:09Z","started_at":"2026-09-25T15:21:09Z","completed_at":"2026-09-25T15:30:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T15:49:35.722030763Z","timing_metadata_checked_at":"2026-09-25T15:49:35.722030763Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-25T15:49:35.725704451Z","region_metadata_checked_at":"2026-09-25T16:52:10.737551819Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"bb229438361a0a19590d5276b3c931aab1b9d725032991d35265e74ba415a7c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103505090173210624","occurred_at":"2026-09-25T15:21:09Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103504024404758528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"open","pr_sha":"7b97dd56e1a77f3691c218ccb60e427185cf8340","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T15:16:55Z","started_at":"2026-09-25T15:16:55Z","completed_at":"2026-09-25T15:25:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T15:39:03.925155429Z","timing_metadata_checked_at":"2026-09-25T15:39:03.925155429Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-25T15:39:03.928695719Z","region_metadata_checked_at":"2026-09-25T16:41:34.70740315Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"92611ca1129c8732a4c696adf3969c000a64d1abcf83da8ef391d147a29ecaa6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103504024404758528","occurred_at":"2026-09-25T15:16:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103499161436426240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7168,"pr_state":"closed","pr_sha":"51f1f4e7727e46a350a13c61b8a6643f5409e63a","final_merged_sha":"438a454514812b6dc5eb79970e7dce00ce113844","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T14:57:30Z","started_at":"2026-09-25T14:57:30Z","completed_at":"2026-09-25T15:15:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T16:09:51.502448373Z","timing_metadata_checked_at":"2026-09-25T16:09:51.502448373Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-25T16:09:51.510998147Z","region_metadata_checked_at":"2026-09-25T17:13:32.532337489Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"15749673366f39edb46c62e25c95fc6716db49d9ae16aa4183c182dda0c3a61c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103499161436426240","occurred_at":"2026-09-25T14:57:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103498474312962048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"3cae25cd3a41cc5575bd878ac507316389d59541","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-25T14:54:51Z","started_at":"2026-09-25T14:54:51Z","completed_at":"2026-09-25T17:10:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T17:23:56.604016403Z","timing_metadata_checked_at":"2026-09-25T17:23:56.604016403Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T17:23:56.607397993Z","region_metadata_checked_at":"2026-09-25T17:23:56.607397993Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103498136960897024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"63e09d878b205c66ece372fb26acee77c5b16bbc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T14:53:31Z","started_at":"2026-09-25T14:53:31Z","completed_at":"2026-09-25T15:05:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T15:17:26.2232139Z","timing_metadata_checked_at":"2026-09-25T15:17:26.2232139Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-25T15:17:26.227069889Z","region_metadata_checked_at":"2026-09-25T16:20:24.520773539Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2e9c025d68920461e48caddff8e7771379a9b32d95864b8fd59c3732f04280d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103498136960897024","occurred_at":"2026-09-25T14:53:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103496438964031488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"2add9465a210f4affb36929a8284f0ccec270352","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T14:46:46Z","started_at":"2026-09-25T14:46:46Z","completed_at":"2026-09-25T14:58:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T15:17:26.48223337Z","timing_metadata_checked_at":"2026-09-25T15:17:26.48223337Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-25T15:17:26.485122861Z","region_metadata_checked_at":"2026-09-25T16:20:26.650464558Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"57e555a3e31785f392466d2866fc3a4cf3ae2afd45ccaa5961afe3e257b681f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103496438964031488","occurred_at":"2026-09-25T14:46:46Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103492665126424576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7146,"pr_state":"closed","pr_sha":"34a27988a2da6ca8b624fbab48af075e636b7eb6","final_merged_sha":"34a27988a2da6ca8b624fbab48af075e636b7eb6","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T14:31:46Z","started_at":"2026-09-25T14:31:46Z","completed_at":"2026-09-25T16:46:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T17:03:07.217340359Z","timing_metadata_checked_at":"2026-09-25T17:03:07.217340359Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T17:03:07.220925048Z","region_metadata_checked_at":"2026-09-25T17:03:07.220925048Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a20a84cb55ed6b5707561ec4db2ad414a971edfc5eddccc559b25e091a815795","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103492665126424576","occurred_at":"2026-09-25T14:31:46Z","failed_at":"alert","signature_id":"9fa96b5ff725dfc0ca78e5fc48314cb4f2f8426a2fe124c1cb522fbfaa4d32f6","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T15:50:58Z\n Ended: 2026-09-25T16:00:56Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6424576-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6424576-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7157,"pr_state":"closed","pr_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","final_merged_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T14:23:20Z","started_at":"2026-09-25T14:23:20Z","completed_at":"2026-09-25T16:39:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T16:52:10.563923656Z","timing_metadata_checked_at":"2026-09-25T16:52:10.563923656Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T16:52:10.566977846Z","region_metadata_checked_at":"2026-09-25T16:52:10.566977846Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"1a520acae5e1ae33c19c882212cc6cc17439f2ec4967970703e03141a828051b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","occurred_at":"2026-09-25T14:23:20Z","failed_at":"alert","signature_id":"53d2828d48a06e30d3c8c55f9cf1eb466eebe159ecb450ed5d78e68b07926aec","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T15:53:12Z\n Ended: 2026-09-25T15:59:09Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9080192-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.97:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9080192/providers/microsoft.monitor/accounts/hcps-j9080192\", namespace=\"ocm-arohcpci01-2t244r5142spe6g2cg9o8mae3d7hdnhu-t6n2x3b5n5t1m1v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-l2hch\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t244r5142spe6g2cg9o8mae3d7hdnhu-t6n2x3b5n5t1m1v/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-25T15:53:12Z\n Ended: 2026-09-25T15:59:09Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9080192-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.97:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9080192/providers/microsoft.monitor/accounts/hcps-j9080192\", namespace=\"ocm-arohcpci01-2t244r5142spe6g2cg9o8mae3d7hdnhu-t6n2x3b5n5t1m1v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-l2hch\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t244r5142spe6g2cg9o8mae3d7hdnhu-t6n2x3b5n5t1m1v/router has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"222639a949b9f7667594911197b3ed5e38a23f5755696fabf41cf6e94d4122df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","occurred_at":"2026-09-25T14:23:20Z","failed_at":"e2e","signature_id":"f1b09e484cd05c31b9ead39590e9ccaac7124eca4de0e096a44d8761ec49c147","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-tql9hwr48v4l/cluster-candidate-4-20-bwhjc4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a9c160\u003e: \n failed to create HCP cluster cluster-candidate-4-20-bwhjc4: failed waiting for cluster=\"cluster-candidate-4-20-bwhjc4\" in resourcegroup=\"rg-candidate-4-20-tql9hwr48v4l\" to finish creating: GET https://rp.j9080192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3415f09-2ec9-41e2-b5a8-2ceeb8dd3ddb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3415f09-2ec9-41e2-b5a8-2ceeb8dd3ddb\",\n \"name\": \"d3415f09-2ec9-41e2-b5a8-2ceeb8dd3ddb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:16:54.769562709Z\",\n \"endTime\": \"2026-09-25T15:46:02.395056967Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.39 is Partial (want Completed), started 23m51s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-tql9hwr48v4l/cluster-candidate-4-20-bwhjc4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a9c160\u003e: \n failed to create HCP cluster cluster-candidate-4-20-bwhjc4: failed waiting for cluster=\"cluster-candidate-4-20-bwhjc4\" in resourcegroup=\"rg-candidate-4-20-tql9hwr48v4l\" to finish creating: GET https://rp.j9080192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3415f09-2ec9-41e2-b5a8-2ceeb8dd3ddb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d3415f09-2ec9-41e2-b5a8-2ceeb8dd3ddb\",\n \"name\": \"d3415f09-2ec9-41e2-b5a8-2ceeb8dd3ddb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:16:54.769562709Z\",\n \"endTime\": \"2026-09-25T15:46:02.395056967Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.39 is Partial (want Completed), started 23m51s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"54968a9a295d0f3d284b5fb31af28de88778dba47cfcb98bd13e98b3d73d593b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","occurred_at":"2026-09-25T14:23:20Z","failed_at":"e2e","signature_id":"1934fc36111a36c3abccdd522caf0d3a5790367da6815ef25c43f7331a3c0e93","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b48a0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-ms6xhv5kcfxf\" to finish creating: GET https://rp.j9080192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ddffd0dd-0556-4e1c-9602-37c54fdf6ce2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ddffd0dd-0556-4e1c-9602-37c54fdf6ce2\",\n \"name\": \"ddffd0dd-0556-4e1c-9602-37c54fdf6ce2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:16:59.889674248Z\",\n \"endTime\": \"2026-09-25T15:46:02.475687505Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m36s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009b48a0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-ms6xhv5kcfxf\" to finish creating: GET https://rp.j9080192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ddffd0dd-0556-4e1c-9602-37c54fdf6ce2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ddffd0dd-0556-4e1c-9602-37c54fdf6ce2\",\n \"name\": \"ddffd0dd-0556-4e1c-9602-37c54fdf6ce2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:16:59.889674248Z\",\n \"endTime\": \"2026-09-25T15:46:02.475687505Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m36s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9eef63e800e0dfe6464d1b5b9c6b6dec29808119d5637a5ee715707e7e467cff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","occurred_at":"2026-09-25T14:23:20Z","failed_at":"e2e","signature_id":"043a7397f094ec76e0bf01b80760be01e69af512654f23e6e4da9739a3ccfa96","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-8vfwxckt8249/cluster-candidate-4-23-pksrt4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc4780\u003e: \n failed to create HCP cluster cluster-candidate-4-23-pksrt4: failed waiting for cluster=\"cluster-candidate-4-23-pksrt4\" in resourcegroup=\"rg-candidate-4-23-8vfwxckt8249\" to finish creating: GET https://rp.j9080192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d480bcb-375c-4b16-8bda-6aad2a19dfa7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d480bcb-375c-4b16-8bda-6aad2a19dfa7\",\n \"name\": \"2d480bcb-375c-4b16-8bda-6aad2a19dfa7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:16:44.436730712Z\",\n \"endTime\": \"2026-09-25T15:45:52.413815981Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.0 is Partial (want Completed), started 23m23s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-8vfwxckt8249/cluster-candidate-4-23-pksrt4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc4780\u003e: \n failed to create HCP cluster cluster-candidate-4-23-pksrt4: failed waiting for cluster=\"cluster-candidate-4-23-pksrt4\" in resourcegroup=\"rg-candidate-4-23-8vfwxckt8249\" to finish creating: GET https://rp.j9080192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d480bcb-375c-4b16-8bda-6aad2a19dfa7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d480bcb-375c-4b16-8bda-6aad2a19dfa7\",\n \"name\": \"2d480bcb-375c-4b16-8bda-6aad2a19dfa7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:16:44.436730712Z\",\n \"endTime\": \"2026-09-25T15:45:52.413815981Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.0 is Partial (want Completed), started 23m23s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dc5010954f35da4001236afe626b55c42eb6d156f9e0cdb6cba1e4c36d0ae39e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","occurred_at":"2026-09-25T14:23:20Z","failed_at":"alert","signature_id":"c3527bcea856a29f65c003d1a332595bb1db06adcefad7894814a3ada10903a6","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T15:48:10Z\n Ended: 2026-09-25T15:59:07Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9080192-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9080192-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"f4a75b09134a096696f4d5a641ba30eecf6858eed9d7e72ccc52060fa633953d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","occurred_at":"2026-09-25T14:23:20Z","failed_at":"alert","signature_id":"1e74d40e28446d51c4e5d1116f6b0a7f045854ce87bcd798968ad60cec419685","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T15:28:09Z\n Ended: 2026-09-25T15:59:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9080192-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t244r5142spe6g2cg9o8mae3d7hdnhu-t6n2x3b5n5t1m1v\", pod=\"router-5664b4f57c-gtjxn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t244r5142spe6g2cg9o8mae3d7hdnhu-t6n2x3b5n5t1m1v/router-5664b4f57c-gtjxn has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-25T15:28:09Z\n Ended: 2026-09-25T15:58:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9080192-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t244nbkhcsoo0pc5tudo89g01hj3nvr-z3c3n6b0x0c8b7t\", pod=\"router-7588d494ff-mm7tn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t244nbkhcsoo0pc5tudo89g01hj3nvr-z3c3n6b0x0c8b7t/router-7588d494ff-mm7tn has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-25T15:30:08Z\n Ended: 2026-09-25T15:59:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9080192-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t244qbe5vrekus0gjf8imef3oj3a1ab-v7v0u3y9z5d5a0r\", pod=\"router-6ffb8bdb9f-rtb5z\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t244qbe5vrekus0gjf8imef3oj3a1ab-v7v0u3y9z5d5a0r/router-6ffb8bdb9f-rtb5z has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103489841365782528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7142,"pr_state":"closed","pr_sha":"cfda26d7698257f0afa1b25f95e39ab26e8f37ef","final_merged_sha":"cfda26d7698257f0afa1b25f95e39ab26e8f37ef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T14:20:33Z","started_at":"2026-09-25T14:20:33Z","completed_at":"2026-09-25T16:28:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T16:41:34.594719278Z","timing_metadata_checked_at":"2026-09-25T16:41:34.594719278Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T16:41:34.598447967Z","region_metadata_checked_at":"2026-09-25T16:41:34.598447967Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"66380dbfe043e7aaa178532817a71234795bfaa293b57f678eef602aab5a001f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103489841365782528","occurred_at":"2026-09-25T14:20:33Z","failed_at":"e2e","signature_id":"035d7dc1692505183a61563c7c881b11d2dac528da1fe3116304bc75371fc438","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002c1e3f0\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-rrf2jq8hklbh\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rrf2jq8hklbh, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002c1e3f0\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-rrf2jq8hklbh\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rrf2jq8hklbh, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-w8sl5z8nrnnt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-w8sl5z8nrnnt, error: context deadline exceeded"}},{"row_id":"a8c619018a78d9cb1894f6d10c002b843cf666548912244e00d4d6b2db53853b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103489841365782528","occurred_at":"2026-09-25T14:20:33Z","failed_at":"e2e","signature_id":"925b7298c77dbd0a35f322d27ca41279ec294430746a196070c77c282811bc46","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00107f620\u003e: \n failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-prrspb8v2jt2\" to finish creating: GET https://rp.j5782528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6fcda38e-d974-452c-817b-05b6eea1a211\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6fcda38e-d974-452c-817b-05b6eea1a211\",\n \"name\": \"6fcda38e-d974-452c-817b-05b6eea1a211\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:19:00.432881696Z\",\n \"endTime\": \"2026-09-25T15:38:00.984242406Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-fh485-cm889: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-prrspb8v2jt2--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-fh485-cm889 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-fh485-cm889: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-prrspb8v2jt2--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-fh485-cm889 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00107f620\u003e: \n failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-prrspb8v2jt2\" to finish creating: GET https://rp.j5782528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6fcda38e-d974-452c-817b-05b6eea1a211\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6fcda38e-d974-452c-817b-05b6eea1a211\",\n \"name\": \"6fcda38e-d974-452c-817b-05b6eea1a211\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:19:00.432881696Z\",\n \"endTime\": \"2026-09-25T15:38:00.984242406Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-fh485-cm889: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-prrspb8v2jt2--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-fh485-cm889 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-fh485-cm889: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-prrspb8v2jt2--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-fh485-cm889 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103489696960090112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7074,"pr_state":"open","pr_sha":"96d277dfa013bc1151443f1a591a57442af16c82","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T14:19:59Z","started_at":"2026-09-25T14:19:59Z","completed_at":"2026-09-25T16:37:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T16:52:10.461528972Z","timing_metadata_checked_at":"2026-09-25T16:52:10.461528972Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T16:52:10.46552406Z","region_metadata_checked_at":"2026-09-25T16:52:10.46552406Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"59c840ed6c1019cc5868b6587fa76d40d23117bef31fb18d0bbc9245d98f27ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103489696960090112","occurred_at":"2026-09-25T14:19:59Z","failed_at":"alert","signature_id":"00ba874216a5343073a0adac3c644d98cfde892f61797b31ee3336d0f5f3c57e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T15:34:43Z\n Ended: 2026-09-25T15:41:44Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0090112-svc\", component=\"backend\", name=\"createnodepoolscopedreaddesires\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue createnodepoolscopedreaddesires has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103487715302445056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"f7eae70a7727145890f4c333ef421260214cc407","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T14:12:06Z","started_at":"2026-09-25T14:12:06Z","completed_at":"2026-09-25T16:21:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T16:31:06.639521384Z","timing_metadata_checked_at":"2026-09-25T16:31:06.639521384Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T16:31:06.642773873Z","region_metadata_checked_at":"2026-09-25T16:31:06.642773873Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"62e1cdf5ae301230a73af453f05030ca8ccbb18530d107c390a591c5a3fe672d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103487715302445056","occurred_at":"2026-09-25T14:12:06Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T15:11:22Z\n Ended: 2026-09-25T15:24:29Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2445056-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"ac523674701207f2068fa6a2a6377b5d9b91b66131fde4ead9bdcad36a705ae7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103487715302445056","occurred_at":"2026-09-25T14:12:06Z","failed_at":"e2e","signature_id":"5981f6c179bac3c28f9dc2c5df2c830c2d41379ed5d4dbea9adf66ba182514a6","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1a8c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-r2tsxmn2gn6l\" to finish creating: GET https://rp.j2445056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d1d1b0c-beb5-491f-977f-de85fe072646\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d1d1b0c-beb5-491f-977f-de85fe072646\",\n \"name\": \"1d1d1b0c-beb5-491f-977f-de85fe072646\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:02:51.1208565Z\",\n \"endTime\": \"2026-09-25T15:21:56.904399075Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-x7jf2-994zl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20260901-np-1-x7jf2-4qp6m: Failed: interfaces failed to create or update. err: failed to get existing resource v20260901preview-r2tsxmn2gn6l--managed/v20260901-np-1-x7jf2-4qp6m-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-r2tsxmn2gn6l--managed/providers/Microsoft.Network/networkInterfaces/v20260901-np-1-x7jf2-4qp6m-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.m... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1a8c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-r2tsxmn2gn6l\" to finish creating: GET https://rp.j2445056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d1d1b0c-beb5-491f-977f-de85fe072646\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d1d1b0c-beb5-491f-977f-de85fe072646\",\n \"name\": \"1d1d1b0c-beb5-491f-977f-de85fe072646\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:02:51.1208565Z\",\n \"endTime\": \"2026-09-25T15:21:56.904399075Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-x7jf2-994zl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20260901-np-1-x7jf2-4qp6m: Failed: interfaces failed to create or update. err: failed to get existing resource v20260901preview-r2tsxmn2gn6l--managed/v20260901-np-1-x7jf2-4qp6m-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-r2tsxmn2gn6l--managed/providers/Microsoft.Network/networkInterfaces/v20260901-np-1-x7jf2-4qp6m-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.m... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Network","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b73f6830c9991fd864cfd1d6c331621eb82a889caa9bac525a376f969499d35c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103487715302445056","occurred_at":"2026-09-25T14:12:06Z","failed_at":"alert","signature_id":"6885dc46c8451b99c4e7c013957eaabbce1c6b219c76ec914f0abe3783c0fde2","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T15:02:29Z\n Ended: 2026-09-25T15:36:28Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2445056-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2445056-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485377787465728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"0ed9e5d46a7c4f7ca3e7d722b7dc05b5b2dacb37","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T14:02:49Z","started_at":"2026-09-25T14:02:49Z","completed_at":"2026-09-25T16:09:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T16:20:24.975581135Z","timing_metadata_checked_at":"2026-09-25T16:20:24.975581135Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T16:20:24.978880625Z","region_metadata_checked_at":"2026-09-25T16:20:24.978880625Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"03dc6792078f2b720d0457d97b8a2c3d8c774a1ba7ff06d91d3cb5c0cb25d378","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485377787465728","occurred_at":"2026-09-25T14:02:49Z","failed_at":"alert","signature_id":"c162a5faff1d028c6c475696bbe945954abd7c58d8b8a99052f72cd3c91b0380","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T15:10:41Z\n Ended: 2026-09-25T15:19:45Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7465728-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"afed8359179d802b8913cdb477f4c8bb3bda97cbdcd2c53100cc78671d4a34de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485377787465728","occurred_at":"2026-09-25T14:02:49Z","failed_at":"e2e","signature_id":"ca14cf116c18f677151c25e3f7d5bfbbd76e0c653dca506e7d80d70050fb3e24","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-mtg998\" for cluster \"cluster-candidate-4-21-mtg998\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0bcc0\u003e: \n failed to create NodePool np-mtg998: failed waiting for nodepool=\"np-mtg998\" for cluster \"cluster-candidate-4-21-mtg998\" in resourcegroup=\"rg-candidate-4-21-6rjxr7bt59b4\" to finish creating: GET https://rp.j7465728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51518391-ba3d-4fba-ad02-341a42607b48\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51518391-ba3d-4fba-ad02-341a42607b48\",\n \"name\": \"51518391-ba3d-4fba-ad02-341a42607b48\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T14:59:49.694049709Z\",\n \"endTime\": \"2026-09-25T15:18:56.27468361Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c9n6t8n1t1l0b2l-np-mtg998-dk6h7-bfvhp: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-21-6rjxr7bt59b4-candidate-mtg998--managed/c9n6t8n1t1l0b2l-np-mtg998-dk6h7-bfvhp-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-6rjxr7bt59b4-candidate-mtg998--managed/providers/Microsoft.Network/networkInterfaces/c9n6t8n1t1l0b2l-np-mtg998-dk6h7-bfvhp-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '3' s... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-mtg998\" for cluster \"cluster-candidate-4-21-mtg998\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0bcc0\u003e: \n failed to create NodePool np-mtg998: failed waiting for nodepool=\"np-mtg998\" for cluster \"cluster-candidate-4-21-mtg998\" in resourcegroup=\"rg-candidate-4-21-6rjxr7bt59b4\" to finish creating: GET https://rp.j7465728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51518391-ba3d-4fba-ad02-341a42607b48\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51518391-ba3d-4fba-ad02-341a42607b48\",\n \"name\": \"51518391-ba3d-4fba-ad02-341a42607b48\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T14:59:49.694049709Z\",\n \"endTime\": \"2026-09-25T15:18:56.27468361Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c9n6t8n1t1l0b2l-np-mtg998-dk6h7-bfvhp: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-21-6rjxr7bt59b4-candidate-mtg998--managed/c9n6t8n1t1l0b2l-np-mtg998-dk6h7-bfvhp-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-6rjxr7bt59b4-candidate-mtg998--managed/providers/Microsoft.Network/networkInterfaces/c9n6t8n1t1l0b2l-np-mtg998-dk6h7-bfvhp-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '3' s... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Network","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485306291359744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7165,"pr_state":"open","pr_sha":"4ada4bfca590fd84b0b6be17befd2127f1df09bf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T14:02:32Z","started_at":"2026-09-25T14:02:32Z","completed_at":"2026-09-25T16:10:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T16:20:25.511318981Z","timing_metadata_checked_at":"2026-09-25T16:20:25.511318981Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T16:20:25.514169672Z","region_metadata_checked_at":"2026-09-25T16:20:25.514169672Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"344e4b4dd1e558617c8e6ae8e2c1ebbced88662643059cb222cc5fffe88943c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485306291359744","occurred_at":"2026-09-25T14:02:32Z","failed_at":"e2e","signature_id":"838df1a6da77eac70a576cc170cfe1289751436f04135f91c74f505e065a568e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rxvsqm\" for cluster \"cluster-candidate-4-23-rxvsqm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d3e620\u003e: \n failed to create NodePool np-rxvsqm: failed waiting for nodepool=\"np-rxvsqm\" for cluster \"cluster-candidate-4-23-rxvsqm\" in resourcegroup=\"rg-candidate-4-23-d278d2sbszjx\" to finish creating: GET https://rp.j1359744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/906078f7-c442-4cef-873d-baedf2876324\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/906078f7-c442-4cef-873d-baedf2876324\",\n \"name\": \"906078f7-c442-4cef-873d-baedf2876324\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:00:06.600392707Z\",\n \"endTime\": \"2026-09-25T15:19:07.477725378Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i0b5a2p5p7p8d7h-np-rxvsqm-tzmkk-tgmgm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-d278d2sbszjx-candidate-rxvsqm--managed/providers/Microsoft.Compute/virtualMachines/i0b5a2p5p7p8d7h-np-rxvsqm-tzmkk-tgmgm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i0b5a2p5p7p8d7h-np-rxvsqm-tzmkk-tgmgm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-d278d2sbszjx-candidate-rxvsqm--managed/providers/Microsoft.Compute/virtualMachines/i0b5a2p5p7p8d7h-np-rxvsqm-tzmkk-tgmgm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rxvsqm\" for cluster \"cluster-candidate-4-23-rxvsqm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d3e620\u003e: \n failed to create NodePool np-rxvsqm: failed waiting for nodepool=\"np-rxvsqm\" for cluster \"cluster-candidate-4-23-rxvsqm\" in resourcegroup=\"rg-candidate-4-23-d278d2sbszjx\" to finish creating: GET https://rp.j1359744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/906078f7-c442-4cef-873d-baedf2876324\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/906078f7-c442-4cef-873d-baedf2876324\",\n \"name\": \"906078f7-c442-4cef-873d-baedf2876324\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:00:06.600392707Z\",\n \"endTime\": \"2026-09-25T15:19:07.477725378Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i0b5a2p5p7p8d7h-np-rxvsqm-tzmkk-tgmgm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-d278d2sbszjx-candidate-rxvsqm--managed/providers/Microsoft.Compute/virtualMachines/i0b5a2p5p7p8d7h-np-rxvsqm-tzmkk-tgmgm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i0b5a2p5p7p8d7h-np-rxvsqm-tzmkk-tgmgm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-d278d2sbszjx-candidate-rxvsqm--managed/providers/Microsoft.Compute/virtualMachines/i0b5a2p5p7p8d7h-np-rxvsqm-tzmkk-tgmgm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c0fb74ec0f32c35361aa3a65833491ae0992f2d96fbea46c0926dce788d386c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485306291359744","occurred_at":"2026-09-25T14:02:32Z","failed_at":"e2e","signature_id":"f069a5566b22b52075f385f50e2f83deea5b73d137ebc80a3f1b3a8567055d32","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc00145e060\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc00145e060\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485064313573376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7172,"pr_state":"closed","pr_sha":"4fe5c2d9e64533d01e53b09460c06df867a07553","final_merged_sha":"2bbccece3c6c89e452cce57250e8ff9917d58772","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T14:01:34Z","started_at":"2026-09-25T14:01:34Z","completed_at":"2026-09-25T16:03:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T16:20:27.002605365Z","timing_metadata_checked_at":"2026-09-25T16:20:27.002605365Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T16:20:27.005354156Z","region_metadata_checked_at":"2026-09-25T16:20:27.005354156Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ceab13d9c96925879d37d13536f20020e53d5ed1a5efb41c78caf11b96e7d32a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485064313573376","occurred_at":"2026-09-25T14:01:34Z","failed_at":"e2e","signature_id":"d41a7f88f22839705a091b2bf0ea143c79226ea00402747e93c227257fef7174","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f73740\u003e: \n failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hmlrrfrqshs5\" to finish creating: GET https://rp.j3573376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f803992-4ae7-4e43-a45d-d0314b8bbda8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f803992-4ae7-4e43-a45d-d0314b8bbda8\",\n \"name\": \"0f803992-4ae7-4e43-a45d-d0314b8bbda8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T14:53:30.12533763Z\",\n \"endTime\": \"2026-09-25T15:12:36.801178206Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c3y8c0z1c8e5r8w-arm64-vm-np-1-92l98-zfsjs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-hmlrrfrqshs5--managed/providers/Microsoft.Compute/virtualMachines/c3y8c0z1c8e5r8w-arm64-vm-np-1-92l98-zfsjs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c3y8c0z1c8e5r8w-arm64-vm-np-1-92l98-zfsjs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-hmlrrfrqshs5--managed/providers/Microsoft.Compute/virtualMachines/c3y8c0z1c8e5r8w-arm64-vm-np-1-92l98-zfsjs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f73740\u003e: \n failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hmlrrfrqshs5\" to finish creating: GET https://rp.j3573376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f803992-4ae7-4e43-a45d-d0314b8bbda8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f803992-4ae7-4e43-a45d-d0314b8bbda8\",\n \"name\": \"0f803992-4ae7-4e43-a45d-d0314b8bbda8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T14:53:30.12533763Z\",\n \"endTime\": \"2026-09-25T15:12:36.801178206Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c3y8c0z1c8e5r8w-arm64-vm-np-1-92l98-zfsjs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-hmlrrfrqshs5--managed/providers/Microsoft.Compute/virtualMachines/c3y8c0z1c8e5r8w-arm64-vm-np-1-92l98-zfsjs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c3y8c0z1c8e5r8w-arm64-vm-np-1-92l98-zfsjs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-hmlrrfrqshs5--managed/providers/Microsoft.Compute/virtualMachines/c3y8c0z1c8e5r8w-arm64-vm-np-1-92l98-zfsjs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7087/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103479903683874816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7087,"pr_state":"closed","pr_sha":"6ef108e37293c8a1a05b1ee1679fda670a90bb09","final_merged_sha":"6ef108e37293c8a1a05b1ee1679fda670a90bb09","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T13:40:24Z","started_at":"2026-09-25T13:40:24Z","completed_at":"2026-09-25T15:50:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T16:09:51.299813398Z","timing_metadata_checked_at":"2026-09-25T16:09:51.299813398Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T16:09:51.302902289Z","region_metadata_checked_at":"2026-09-25T16:09:51.302902289Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7173/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103479547658768384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7173,"pr_state":"open","pr_sha":"d2a1db6b8d09b20e6c5c89934d4f7a877b158a12","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-25T13:39:39Z","started_at":"2026-09-25T13:39:39Z","completed_at":"2026-09-25T15:54:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T16:09:51.781897312Z","timing_metadata_checked_at":"2026-09-25T16:09:51.781897312Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T16:09:51.784772903Z","region_metadata_checked_at":"2026-09-25T16:09:51.784772903Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7171/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103473871385530368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7171,"pr_state":"closed","pr_sha":"746f52fa2410704e21a77c1a90e3dc2c474e23ee","final_merged_sha":"746f52fa2410704e21a77c1a90e3dc2c474e23ee","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T13:17:06Z","started_at":"2026-09-25T13:17:06Z","completed_at":"2026-09-25T15:31:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T15:49:35.741803601Z","timing_metadata_checked_at":"2026-09-25T15:49:35.741803601Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T15:49:35.744263093Z","region_metadata_checked_at":"2026-09-25T15:49:35.744263093Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103470957527044096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T13:05:31Z","started_at":"2026-09-25T13:05:31Z","completed_at":"2026-09-25T15:15:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T15:28:20.875512177Z","timing_metadata_checked_at":"2026-09-25T15:28:20.875512177Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T15:28:20.879750464Z","region_metadata_checked_at":"2026-09-25T15:28:20.879750464Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9fabd1040df93b9d5aa45eced529cdcdea5ba43d66da5cdb23431054efd7343d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103470957527044096","occurred_at":"2026-09-25T13:05:31Z","failed_at":"e2e","signature_id":"3e7ca000732e0f699ce4bc50769e5e56cc9b3b001ef3b4dfb6ec429441e9f0d9","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--nrk2cdj4tcn5/cluster-hs-pre-pcqks7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009409a0\u003e: \n failed to create HCP cluster cluster-hs-pre-pcqks7: failed waiting for cluster=\"cluster-hs-pre-pcqks7\" in resourcegroup=\"rg-hs-pre--nrk2cdj4tcn5\" to finish creating: GET https://rp.j7044096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e89da08a-794d-4885-a19f-3eb28678e2ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e89da08a-794d-4885-a19f-3eb28678e2ea\",\n \"name\": \"e89da08a-794d-4885-a19f-3eb28678e2ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T13:47:52.911662059Z\",\n \"endTime\": \"2026-09-25T14:16:59.018766333Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--nrk2cdj4tcn5/cluster-hs-pre-pcqks7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009409a0\u003e: \n failed to create HCP cluster cluster-hs-pre-pcqks7: failed waiting for cluster=\"cluster-hs-pre-pcqks7\" in resourcegroup=\"rg-hs-pre--nrk2cdj4tcn5\" to finish creating: GET https://rp.j7044096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e89da08a-794d-4885-a19f-3eb28678e2ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e89da08a-794d-4885-a19f-3eb28678e2ea\",\n \"name\": \"e89da08a-794d-4885-a19f-3eb28678e2ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T13:47:52.911662059Z\",\n \"endTime\": \"2026-09-25T14:16:59.018766333Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e4a7d5ab159da45fe5b8531c23a4be8200ae4c86e143d0f8fe893280bd62d2b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103470957527044096","occurred_at":"2026-09-25T13:05:31Z","failed_at":"alert","signature_id":"b3ccd27aced22d80e0151b4a766369a088986df8831ba4c138ed86573b3162d6","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T14:11:33Z\n Ended: 2026-09-25T14:34:32Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7044096-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.121:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7044096/providers/microsoft.monitor/accounts/hcps-j7044096\", namespace=\"ocm-arohcpci01-2t22r2i2q615199mla8363r7atbm0o71-z4i0x9u5g9q8w6e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-m9x86\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t22r2i2q615199mla8363r7atbm0o71-z4i0x9u5g9q8w6e/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103470931874680832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"e207b76e733cb65c677c16c7f38706c45baff4cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-25T13:05:25Z","started_at":"2026-09-25T13:05:25Z","completed_at":"2026-09-25T15:16:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T15:28:20.703866728Z","timing_metadata_checked_at":"2026-09-25T15:28:20.703866728Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T15:28:20.708221114Z","region_metadata_checked_at":"2026-09-25T15:28:20.708221114Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7166/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103468512583356416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7166,"pr_state":"closed","pr_sha":"9f8dae2722e980002acb8a47fa4e8e18091d6054","final_merged_sha":"9f8dae2722e980002acb8a47fa4e8e18091d6054","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T12:55:48Z","started_at":"2026-09-25T12:55:48Z","completed_at":"2026-09-25T15:05:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T15:17:26.358395019Z","timing_metadata_checked_at":"2026-09-25T15:17:26.358395019Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T15:17:26.36162251Z","region_metadata_checked_at":"2026-09-25T15:17:26.36162251Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7155/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103462425171857408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7155,"pr_state":"closed","pr_sha":"2c285c27d181371596fd7063d957601efb266472","final_merged_sha":"b99e022318555865ffcb4306017cec183e68d902","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T12:31:37Z","started_at":"2026-09-25T12:31:37Z","completed_at":"2026-09-25T14:35:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T14:47:01.695207088Z","timing_metadata_checked_at":"2026-09-25T14:47:01.695207088Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T14:47:01.69783368Z","region_metadata_checked_at":"2026-09-25T14:47:01.69783368Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"9e36936e417f7ae304e29fdc7c24251464d4d2db2bd10f2c4cda7dd45dcce269","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7155/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103462425171857408","occurred_at":"2026-09-25T12:31:37Z","failed_at":"alert","signature_id":"028386accf8ed10d4b0a82a3fc25eda87277e1cd283cfc4e15e27e0fae5cc1db","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T13:53:05Z\n Ended: 2026-09-25T14:01:07Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1857408-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1857408-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103461761565855744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"closed","pr_sha":"90c3b8a82bebade41bd21c89202388c54efd1a17","final_merged_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T12:28:58Z","started_at":"2026-09-25T12:28:58Z","completed_at":"2026-09-25T14:47:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T15:07:12.568461609Z","timing_metadata_checked_at":"2026-09-25T15:07:12.568461609Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T15:07:12.572044798Z","region_metadata_checked_at":"2026-09-25T15:07:12.572044798Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"3ff6e2dd017d0651c408fe24706ffe044f214a2fe547846ab4cd2e351b422e16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103461761565855744","occurred_at":"2026-09-25T12:28:58Z","failed_at":"alert","signature_id":"9dfe5be11daafcf9e81f7d064effd8fc3de0a5d41bf293ebd6005bf43e5445b0","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T13:54:17Z\n Ended: 2026-09-25T14:01:17Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5855744-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5855744-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"c18fd7d4707bd3eea95409c6ce7c8e223b3784d226ec5087d056c2ffb9ad0f26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103461761565855744","occurred_at":"2026-09-25T12:28:58Z","failed_at":"alert","signature_id":"16462dfb1b7ae544495c5b7da4413d3ef0ee0bc77c796be4c8df88ca7c8dbf0a","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T13:29:51Z\n Ended: 2026-09-25T13:50:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5855744-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t22eefold2tc0okjf84jebs6os67poh-f3x9e9n2u8k1f3x\", pod=\"router-866f6b9b68-5nwqm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t22eefold2tc0okjf84jebs6os67poh-f3x9e9n2u8k1f3x/router-866f6b9b68-5nwqm has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7163/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103461242604621824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7163,"pr_state":"closed","pr_sha":"e1e2c7572fd4860250f0893f8b53c154929b9e83","final_merged_sha":"e1e2c7572fd4860250f0893f8b53c154929b9e83","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T12:26:55Z","started_at":"2026-09-25T12:26:55Z","completed_at":"2026-09-25T14:39:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T14:47:01.851466872Z","timing_metadata_checked_at":"2026-09-25T14:47:01.851466872Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T14:47:01.856235857Z","region_metadata_checked_at":"2026-09-25T14:47:01.856235857Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a210342de2cb20b5ca575cae01a394d758e1aaefc55b8368b166fcab860c2170","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7163/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103461242604621824","occurred_at":"2026-09-25T12:26:55Z","failed_at":"e2e","signature_id":"b45947755f19be26b32e2accffc4cc0be60dad387febd577240f60063bf46a38","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00085dbd8\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-566qcntr7gdn\" to finish creating: GET https://rp.j4621824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47438056-285b-41f2-ab31-9d3620f43054\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47438056-285b-41f2-ab31-9d3620f43054\",\n \"name\": \"47438056-285b-41f2-ab31-9d3620f43054\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T13:28:05.551349476Z\",\n \"endTime\": \"2026-09-25T13:47:09.734640501Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-h9wwd-6cst6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-566qcntr7gdn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-h9wwd-6cst6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-h9wwd-6cst6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-566qcntr7gdn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-h9wwd-6cst6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n encountered errors downloading console logs:\n - VM \"cilium-cluster-cilium-np-h9wwd-6cst6\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-h9wwd-6cst6\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-566qcntr7gdn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-h9wwd-6cst6/retrieveBootDiagnosticsData\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: OperationNotAllowed\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"OperationNotAllowed\",\n \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-h9wwd-6cst6' since another operation is in progress.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00085dbd8\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-566qcntr7gdn\" to finish creating: GET https://rp.j4621824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47438056-285b-41f2-ab31-9d3620f43054\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47438056-285b-41f2-ab31-9d3620f43054\",\n \"name\": \"47438056-285b-41f2-ab31-9d3620f43054\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T13:28:05.551349476Z\",\n \"endTime\": \"2026-09-25T13:47:09.734640501Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-h9wwd-6cst6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-566qcntr7gdn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-h9wwd-6cst6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-h9wwd-6cst6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-566qcntr7gdn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-h9wwd-6cst6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n encountered errors downloading console logs:\n - VM \"cilium-cluster-cilium-np-h9wwd-6cst6\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-h9wwd-6cst6\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-566qcntr7gdn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-h9wwd-6cst6/retrieveBootDiagnosticsData\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: OperationNotAllowed\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"OperationNotAllowed\",\n \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-h9wwd-6cst6' since another operation is in progress.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ddfc035a80bbbf2ad88605092a69732d98a1147956895ca4acb58224e5834fe","failure_pattern":"ERROR CODE: InternalServerError; detail code OperationNotAllowed; detail message Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-\u003cid\u003e' since another operation is in progress.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6ddfc035a80bbbf2ad88605092a69732d98a1147956895ca4acb58224e5834fe"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7123,"pr_state":"closed","pr_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","final_merged_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T12:24:21Z","started_at":"2026-09-25T12:24:21Z","completed_at":"2026-09-25T14:38:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T14:47:01.549884068Z","timing_metadata_checked_at":"2026-09-25T14:47:01.549884068Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T14:47:01.554947251Z","region_metadata_checked_at":"2026-09-25T14:47:01.554947251Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"0adb3497f290555a2ac0e5825420e66db0264d43af475311905160c0f5de1ef5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","occurred_at":"2026-09-25T12:24:21Z","failed_at":"alert","signature_id":"ec90986a9952136621543366f85ab08fb1261aa887e4625683d776a59dc714cd","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T13:51:55Z\n Ended: 2026-09-25T13:59:55Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5675648-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5675648-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"51a7f2a01880bf601102e47c2d0dd0dda3511a8effafde1f9e491ff26acf5b4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","occurred_at":"2026-09-25T12:24:21Z","failed_at":"e2e","signature_id":"b09aada1d36d2ce5d689faef6eae5346e1bed7ccb7417af23f7078fe3705cf6e","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000d56060\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-ljjsv8j98zgt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-ljjsv8j98zgt, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000d56060\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-ljjsv8j98zgt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-ljjsv8j98zgt, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-w8sl5z8nrnnt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-w8sl5z8nrnnt, error: context deadline exceeded"}},{"row_id":"54f8f98505616d3ae1ebe5bc4f8c1d315dfc97c2559af4372b37d820c7a6fd9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","occurred_at":"2026-09-25T12:24:21Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T13:32:07Z\n Ended: 2026-09-25T13:43:07Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5675648-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"73e772248613b8d5aa8553f23686e770878fe35165aa4cdb87e612b5fc1395cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","occurred_at":"2026-09-25T12:24:21Z","failed_at":"alert","signature_id":"0a0c47e7fa898ae28c5add2df96849779c38742a25e77d6ee2c158a5dcead973","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T13:27:57Z\n Ended: 2026-09-25T13:56:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5675648-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t22c27eesbt60en98i4csdbemrlhr44-v20261001\", pod=\"router-db8fb6c46-6jtmm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t22c27eesbt60en98i4csdbemrlhr44-v20261001/router-db8fb6c46-6jtmm has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-25T13:41:56Z\n Ended: 2026-09-25T13:47:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5675648-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t22cvuckounk62fjihgg1h5pnaiu1h5-y1b7j9n3p2c4e0z\", pod=\"konnectivity-agent-7848799974-nnq7f\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t22cvuckounk62fjihgg1h5pnaiu1h5-y1b7j9n3p2c4e0z/konnectivity-agent-7848799974-nnq7f has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"db7be066562ff1ed812fdffe7e947d8b236c4e40692b0a9c34c3f632666a9b11","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","occurred_at":"2026-09-25T12:24:21Z","failed_at":"e2e","signature_id":"e27c2b8a568a4d0f9b1055a191452eb1394a405a2e80a4022d25f6917c8f9fba","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00148e780\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-l6zhdc\" in resourcegroup=\"rg-np-version-upgrade-l6zhdc-jh8xxsghsgf8\" to finish creating: GET https://rp.j5675648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8325ef8e-dd78-4b1c-8d7e-a8b5d077aa5a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8325ef8e-dd78-4b1c-8d7e-a8b5d077aa5a\",\n \"name\": \"8325ef8e-dd78-4b1c-8d7e-a8b5d077aa5a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T13:29:34.053488486Z\",\n \"endTime\": \"2026-09-25T13:48:39.892100532Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-l6zhdc-jh8xxsghsgf8-np-upgrade-l6-78bab067/providers/Microsoft.Compute/virtualMachines/f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-l6zhdc-jh8xxsghsgf8-np-upgrade-l6-78bab067/providers/Microsoft.Compute/virtualMachines/f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00148e780\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-l6zhdc\" in resourcegroup=\"rg-np-version-upgrade-l6zhdc-jh8xxsghsgf8\" to finish creating: GET https://rp.j5675648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8325ef8e-dd78-4b1c-8d7e-a8b5d077aa5a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8325ef8e-dd78-4b1c-8d7e-a8b5d077aa5a\",\n \"name\": \"8325ef8e-dd78-4b1c-8d7e-a8b5d077aa5a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T13:29:34.053488486Z\",\n \"endTime\": \"2026-09-25T13:48:39.892100532Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-l6zhdc-jh8xxsghsgf8-np-upgrade-l6-78bab067/providers/Microsoft.Compute/virtualMachines/f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-l6zhdc-jh8xxsghsgf8-np-upgrade-l6-78bab067/providers/Microsoft.Compute/virtualMachines/f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eacd13fa6af2b06f56ce1295197593af50aee51808239f944fd6ca6273414a50","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e2a69a88afdcec58169aedd6f98d9cf27b9e2adc57ba8483edbe1b28731cc584","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","occurred_at":"2026-09-25T12:24:21Z","failed_at":"e2e","signature_id":"080ef2ca65397bcb277ef88227d88a290162f457dffe508d8b1669032d36145a","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000990ae0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-wf95j7vx69b5\" to finish creating: GET https://rp.j5675648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00381033-aeb7-400a-bd07-e1bd8611cea2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00381033-aeb7-400a-bd07-e1bd8611cea2\",\n \"name\": \"00381033-aeb7-400a-bd07-e1bd8611cea2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T13:15:51.082389682Z\",\n \"endTime\": \"2026-09-25T13:45:00.023728639Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000990ae0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-wf95j7vx69b5\" to finish creating: GET https://rp.j5675648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00381033-aeb7-400a-bd07-e1bd8611cea2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00381033-aeb7-400a-bd07-e1bd8611cea2\",\n \"name\": \"00381033-aeb7-400a-bd07-e1bd8611cea2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T13:15:51.082389682Z\",\n \"endTime\": \"2026-09-25T13:45:00.023728639Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","eacd13fa6af2b06f56ce1295197593af50aee51808239f944fd6ca6273414a50"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7131/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460624594898944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7131,"pr_state":"closed","pr_sha":"cb8c2640a77a75ae501a0a609a7518ef1807d76b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-25T12:24:20Z","started_at":"2026-09-25T12:24:20Z","completed_at":"2026-09-25T14:45:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T14:57:07.300616869Z","timing_metadata_checked_at":"2026-09-25T14:57:07.300616869Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T14:57:07.304235057Z","region_metadata_checked_at":"2026-09-25T14:57:07.304235057Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460622090899456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7133,"pr_state":"open","pr_sha":"c2d1719d2c83ce4eb02f5e7803f798b5b01e417f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T12:24:19Z","started_at":"2026-09-25T12:24:19Z","completed_at":"2026-09-25T13:19:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T13:30:13.73451241Z","timing_metadata_checked_at":"2026-09-25T13:30:13.73451241Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T13:30:13.738013099Z","region_metadata_checked_at":"2026-09-25T13:30:13.738013099Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"600190f2652c50e4c7c25498c41b84e586e33d7f72bb5bce142bdcd32e5e2b96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460622090899456","occurred_at":"2026-09-25T12:24:19Z","failed_at":"provision","signature_id":"3eea31dd2ba80ae3c26eda9638a5c4961c637f75b7b51fe4771e43332561b111","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T13:12:26.954Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-09-25T13:12:27.437Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4]\"\ntime=2026-09-25T13:12:27.438Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-09-25T13:12:27.610Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 namespace=aro-hcp-mitigations\ntime=2026-09-25T13:12:27.765Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 dryRun=false\ntime=2026-09-25T13:12:27.765Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-09-25T13:12:27.816Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-09-25T13:13:20.164Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-09-25T13:13:20.164Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7155/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103457524530286592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7155,"pr_state":"closed","pr_sha":"2c285c27d181371596fd7063d957601efb266472","final_merged_sha":"b99e022318555865ffcb4306017cec183e68d902","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T12:12:08Z","started_at":"2026-09-25T12:12:08Z","completed_at":"2026-09-25T12:22:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T12:36:19.826588816Z","timing_metadata_checked_at":"2026-09-25T12:36:19.826588816Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-25T12:36:19.829079608Z","region_metadata_checked_at":"2026-09-25T13:40:35.714332705Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0ff5bc50941e26b7e3ce941e4212baf0b3a504e12d5662a29b7f5fb8a88e0079","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7155/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103457524530286592","occurred_at":"2026-09-25T12:12:08Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103457038066520064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"3cae25cd3a41cc5575bd878ac507316389d59541","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T12:10:12Z","started_at":"2026-09-25T12:10:12Z","completed_at":"2026-09-25T14:28:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T14:44:40.353558575Z","timing_metadata_checked_at":"2026-09-25T14:44:40.353558575Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T14:44:40.363661242Z","region_metadata_checked_at":"2026-09-25T14:44:40.363661242Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ec78e29596436b719cc3b4f68558fbf9ba8ab98eec194d15a3452ddd0bcb7e81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103457038066520064","occurred_at":"2026-09-25T12:10:12Z","failed_at":"e2e","signature_id":"a46ff91bb9b72d4ce7de86d214d11c025bf439cd09bdf0a909c8fdbf3a4d558f","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00154c2a0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-w8sl5z8nrnnt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-w8sl5z8nrnnt, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00154c2a0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-w8sl5z8nrnnt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-w8sl5z8nrnnt, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-w8sl5z8nrnnt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-w8sl5z8nrnnt, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103451563560275968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"f7eae70a7727145890f4c333ef421260214cc407","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T11:48:27Z","started_at":"2026-09-25T11:48:27Z","completed_at":"2026-09-25T13:52:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T14:02:06.335131832Z","timing_metadata_checked_at":"2026-09-25T14:02:06.335131832Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T14:02:06.338467521Z","region_metadata_checked_at":"2026-09-25T14:02:06.338467521Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"0353bee10701acbdd03f456f6f2813585d09666a1c2db6f8caf20ab335d737d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103451563560275968","occurred_at":"2026-09-25T11:48:27Z","failed_at":"e2e","signature_id":"1b01d9e119adc9dde05f5270a026a56addb171a7b99ea7686bbcfe2a7ca2162d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:136]: failed to get admin REST config for cluster \"cluster-zstream-4-21-kz5vz4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010728c0\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-21-kz5vz4\" in resourcegroup=\"rg-zstream-upgrade-4-21-xtgkx9mrrcpz\" to finish getting creds: GET https://rp.j0275968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/773f5d4a-cd7a-4e25-ac56-58130cbed2d2\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/773f5d4a-cd7a-4e25-ac56-58130cbed2d2' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/773f5d4a-cd7a-4e25-ac56-58130cbed2d2\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:136]: failed to get admin REST config for cluster \"cluster-zstream-4-21-kz5vz4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010728c0\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-21-kz5vz4\" in resourcegroup=\"rg-zstream-upgrade-4-21-xtgkx9mrrcpz\" to finish getting creds: GET https://rp.j0275968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/773f5d4a-cd7a-4e25-ac56-58130cbed2d2\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/773f5d4a-cd7a-4e25-ac56-58130cbed2d2' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/773f5d4a-cd7a-4e25-ac56-58130cbed2d2\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}},{"row_id":"b2df7a63593219580886c1b002dc5114eb77966b8076310c3da2ef3b4eb9d4d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103451563560275968","occurred_at":"2026-09-25T11:48:27Z","failed_at":"alert","signature_id":"8e84a5dd9589dab0c443e45f6f89b76cc4dfe1440ffcabce2df34ccdfef20636","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T13:13:06Z\n Ended: 2026-09-25T13:19:06Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0275968-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0275968-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"d16f4665f0d6528efad548418f86b44be823511bcafd97fad01d7e1690c46836","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103451563560275968","occurred_at":"2026-09-25T11:48:27Z","failed_at":"e2e","signature_id":"12ae308cd1789be846262f9aa7e8cd0ffc37068656f3e5bc64966bcafd3a0dec","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:97]: failed to create customer resources for basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c324a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-cluster-rjv7q9\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-9crmxlzr24l9\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-9crmxlzr24l9/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-rjv7q9/operationStatuses/08584112672274846775\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-xafodxhow6npw' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:97]: failed to create customer resources for basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c324a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-cluster-rjv7q9\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-9crmxlzr24l9\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-9crmxlzr24l9/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-rjv7q9/operationStatuses/08584112672274846775\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-xafodxhow6npw' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"e7de292ddbea297fb557f64cb4c693afb81fbc6cc642e86f2ede07ea7a98c430","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103451563560275968","occurred_at":"2026-09-25T11:48:27Z","failed_at":"alert","signature_id":"ddf8ccdb3122a4b47c1d6e20ff22259362cf0d7b2c3a63d9a23d1d7a7a27f9c9","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T13:02:21Z\n Ended: 2026-09-25T13:08:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0275968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t21pc68s7hu8p9g83tiddmismuebn5j-t8x7t0c9m0s0q5d\", pod=\"ovnkube-control-plane-54857f87-xzxrv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t21pc68s7hu8p9g83tiddmismuebn5j-t8x7t0c9m0s0q5d/ovnkube-control-plane-54857f87-xzxrv has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103450358956167168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7157,"pr_state":"closed","pr_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","final_merged_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T11:43:23Z","started_at":"2026-09-25T11:43:23Z","completed_at":"2026-09-25T14:01:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T14:12:39.300050545Z","timing_metadata_checked_at":"2026-09-25T14:12:39.300050545Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T14:12:39.303547134Z","region_metadata_checked_at":"2026-09-25T14:12:39.303547134Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"8fa09f222fe3a92009547817b1d741afffe4d13f8fdd57330e6ffbe31ee250f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103450358956167168","occurred_at":"2026-09-25T11:43:23Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T12:51:02Z\n Ended: 2026-09-25T13:03:02Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6167168-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"c698bd82c899bc7b96151923b6d8f1bf2314ff25d72f47ed956c5a8112220b2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103450358956167168","occurred_at":"2026-09-25T11:43:23Z","failed_at":"alert","signature_id":"f744a6565b76474ced1ce309d8efc72176260c6843e213d9c3fca66de936d085","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T13:13:25Z\n Ended: 2026-09-25T13:19:27Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6167168-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6167168-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7167/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103448573973630976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7167,"pr_state":"closed","pr_sha":"bc364b193bf9a52ea2ccf6e3e37a8b8f8e41c5bc","final_merged_sha":"bc364b193bf9a52ea2ccf6e3e37a8b8f8e41c5bc","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T11:36:33Z","started_at":"2026-09-25T11:36:33Z","completed_at":"2026-09-25T14:02:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T14:12:39.608137874Z","timing_metadata_checked_at":"2026-09-25T14:12:39.608137874Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T14:12:39.610660266Z","region_metadata_checked_at":"2026-09-25T14:12:39.610660266Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103447011792523264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7165,"pr_state":"open","pr_sha":"4ada4bfca590fd84b0b6be17befd2127f1df09bf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T11:30:21Z","started_at":"2026-09-25T11:30:21Z","completed_at":"2026-09-25T13:59:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T14:12:39.443923191Z","timing_metadata_checked_at":"2026-09-25T14:12:39.443923191Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T14:12:39.446615383Z","region_metadata_checked_at":"2026-09-25T14:12:39.446615383Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"57ba49583cd6266018095f71f3a3aeb0cb113933702396ecd8cce6d54c46597f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103447011792523264","occurred_at":"2026-09-25T11:30:21Z","failed_at":"e2e","signature_id":"101f712af098953d60298de1798c777e135600eee6b011a0c229833e88e62074","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca7e40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-rjcjqxxsgq7f\" to finish creating: GET https://rp.j2523264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17378f74-366a-451c-a16d-8f6eab8588d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17378f74-366a-451c-a16d-8f6eab8588d2\",\n \"name\": \"17378f74-366a-451c-a16d-8f6eab8588d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T12:45:45.224128185Z\",\n \"endTime\": \"2026-09-25T13:04:46.064091322Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-rjcjqxxsgq7f--managed/providers/Microsoft.Compute/virtualMachines/u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-rjcjqxxsgq7f--managed/providers/Microsoft.Compute/virtualMachines/u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca7e40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-rjcjqxxsgq7f\" to finish creating: GET https://rp.j2523264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17378f74-366a-451c-a16d-8f6eab8588d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17378f74-366a-451c-a16d-8f6eab8588d2\",\n \"name\": \"17378f74-366a-451c-a16d-8f6eab8588d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T12:45:45.224128185Z\",\n \"endTime\": \"2026-09-25T13:04:46.064091322Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-rjcjqxxsgq7f--managed/providers/Microsoft.Compute/virtualMachines/u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-rjcjqxxsgq7f--managed/providers/Microsoft.Compute/virtualMachines/u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eacd13fa6af2b06f56ce1295197593af50aee51808239f944fd6ca6273414a50","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aa76ab5aee0d3b011dfc0ca859d7bcd178c6f42336334cb6583ff4c872068668","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103447011792523264","occurred_at":"2026-09-25T11:30:21Z","failed_at":"e2e","signature_id":"3d00756f058ac76508c5214768bdd5e72a8833c6eb5c3c560deb2bc53bc00897","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00134acc0\u003e: \n VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-nfdpj-h6dps]\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-z8btrh6l7hvp\" to finish creating: GET https://rp.j2523264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9dfb13f2-af5d-471c-b272-b1c35505253f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9dfb13f2-af5d-471c-b272-b1c35505253f\",\n \"name\": \"9dfb13f2-af5d-471c-b272-b1c35505253f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T12:51:15.748310239Z\",\n \"endTime\": \"2026-09-25T13:10:18.43513357Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-nfdpj-h6dps: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-z8btrh6l7hvp--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-nfdpj-h6dps to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-nfdpj-h6dps: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-z8btrh6l7hvp--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-nfdpj-h6dps to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00134acc0\u003e: \n VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-nfdpj-h6dps]\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-z8btrh6l7hvp\" to finish creating: GET https://rp.j2523264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9dfb13f2-af5d-471c-b272-b1c35505253f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9dfb13f2-af5d-471c-b272-b1c35505253f\",\n \"name\": \"9dfb13f2-af5d-471c-b272-b1c35505253f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T12:51:15.748310239Z\",\n \"endTime\": \"2026-09-25T13:10:18.43513357Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-nfdpj-h6dps: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-z8btrh6l7hvp--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-nfdpj-h6dps to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-nfdpj-h6dps: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-z8btrh6l7hvp--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-nfdpj-h6dps to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b6695a9c8ed566b40df73975745c0c7f92e4f64ce36419ead0b585fc65511cae","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want \u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-nfdpj-\u003cid\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-nfdpj-\u003cid\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e32ef26294298f69388ec4a221916ff79f1bd8ff55079d1dd122c66240541537","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103447011792523264","occurred_at":"2026-09-25T11:30:21Z","failed_at":"alert","signature_id":"c4612c6f0c4f97aa285084bb11c15657a376a3c30bfeeb5dba0a5261ddcef8d0","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T13:13:21Z\n Ended: 2026-09-25T13:19:19Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2523264-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2523264-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["b6695a9c8ed566b40df73975745c0c7f92e4f64ce36419ead0b585fc65511cae","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","eacd13fa6af2b06f56ce1295197593af50aee51808239f944fd6ca6273414a50"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103435424331468800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"open","pr_sha":"2bea26fa7ecf716e7bb08a353e54aeb6f760bb48","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T10:44:19Z","started_at":"2026-09-25T10:44:19Z","completed_at":"2026-09-25T10:53:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T11:12:41.320191234Z","timing_metadata_checked_at":"2026-09-25T11:12:41.320191234Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-25T11:12:41.323676323Z","region_metadata_checked_at":"2026-09-25T12:14:42.846948936Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a5161a3320ac2455ebbf1a0de5ea6e196f7872b7f72e53d473b2285add383418","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103435424331468800","occurred_at":"2026-09-25T10:44:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103429520567570432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7164,"pr_state":"closed","pr_sha":"653ff4451ef4e6d7031565b7f3e25b8a6be9a485","final_merged_sha":"653ff4451ef4e6d7031565b7f3e25b8a6be9a485","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T10:20:52Z","started_at":"2026-09-25T10:20:52Z","completed_at":"2026-09-25T11:27:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T11:43:33.094080775Z","timing_metadata_checked_at":"2026-09-25T11:43:33.094080775Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T11:43:33.096850367Z","region_metadata_checked_at":"2026-09-25T11:43:33.096850367Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"7a32d19c558b2729c4b55fe161ff54654a8bf861f604f0a08d0988e4cb014e42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103429520567570432","occurred_at":"2026-09-25T10:20:52Z","failed_at":"alert","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-25T11:11:32Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j7570432-mgmt-1\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}},{"row_id":"b41845cfc0e764621481ed113d5c0d9bc40e77d64c115eb73398f086d0ba843f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103429520567570432","occurred_at":"2026-09-25T10:20:52Z","failed_at":"provision","signature_id":"4c0157b6486aa8ba13127ff990f84148fa30b014d23cc1a62b37dca7e1e44aab","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T10:49:06.749Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-25T11:19:06.506Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-25T10:49:06.755889618Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7570432-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7570432-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-09-25T10:49:13.818157295Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7570432-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7570432-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-25T10:49:14.199363511Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7570432-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7570432-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6469a04d9641eaa98fbab1139ee6c156e917f52ffa704cc151fa46e1e328673d","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j7570432-mgmt-2\",\"clusterName\":\"ci01-j757043","search_query":"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-25T10:49:06.755889618Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7570432-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7570432-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-09-25T10:49:13.818157295Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7570432-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7570432-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-25T10:49:14.199363511Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7570432-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7570432-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6469a04d9641eaa98fbab1139ee6c156e917f52ffa704cc151fa46e1e328673d","abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103427021102125056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"open","pr_sha":"9a39d1a5ebaf621b206fab331ccd3298b8449a15","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T10:10:56Z","started_at":"2026-09-25T10:10:56Z","completed_at":"2026-09-25T10:37:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T10:51:44.481597286Z","timing_metadata_checked_at":"2026-09-25T10:51:44.481597286Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T10:51:44.486017972Z","region_metadata_checked_at":"2026-09-25T10:51:44.486017972Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3f2ec5fcfd97a6ee84586866a268f8aa407a5c4f93193d1f32c1c92b9cbd3898","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103427021102125056","occurred_at":"2026-09-25T10:10:56Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103424207261011968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7003,"pr_state":"closed","pr_sha":"5ef6466e2a8ec2041b8b57cd5821ccdc91bbdd88","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T09:59:45Z","started_at":"2026-09-25T09:59:45Z","completed_at":"2026-09-25T12:21:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T12:36:19.011418708Z","timing_metadata_checked_at":"2026-09-25T12:36:19.011418708Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T12:36:19.014076Z","region_metadata_checked_at":"2026-09-25T12:36:19.014076Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"036af0d160793e055708dc506d88059359f6e44d2c2dbf3d17144121a802fc7d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103424207261011968","occurred_at":"2026-09-25T09:59:45Z","failed_at":"e2e","signature_id":"878ef4ab647acb1943f7c9c33318d2974704d5a838a24ec4551ed0902a3582c8","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0007fbbe0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-9ngsk5k5bt5f\" to finish creating: GET https://rp.j1011968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c50dfee1-939c-4199-9413-eca90594ea6c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c50dfee1-939c-4199-9413-eca90594ea6c\",\n \"name\": \"c50dfee1-939c-4199-9413-eca90594ea6c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T11:12:13.33985788Z\",\n \"endTime\": \"2026-09-25T11:31:22.686534566Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z3q2f0g2z5d6n0b-np-delete-main-72qcj-6f66h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-9ngsk5k5bt5f--managed/providers/Microsoft.Compute/virtualMachines/z3q2f0g2z5d6n0b-np-delete-main-72qcj-6f66h to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z3q2f0g2z5d6n0b-np-delete-main-72qcj-6f66h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-9ngsk5k5bt5f--managed/providers/Microsoft.Compute/virtualMachines/z3q2f0g2z5d6n0b-np-delete-main-72qcj-6f66h to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0007fbbe0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-9ngsk5k5bt5f\" to finish creating: GET https://rp.j1011968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c50dfee1-939c-4199-9413-eca90594ea6c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c50dfee1-939c-4199-9413-eca90594ea6c\",\n \"name\": \"c50dfee1-939c-4199-9413-eca90594ea6c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T11:12:13.33985788Z\",\n \"endTime\": \"2026-09-25T11:31:22.686534566Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z3q2f0g2z5d6n0b-np-delete-main-72qcj-6f66h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-9ngsk5k5bt5f--managed/providers/Microsoft.Compute/virtualMachines/z3q2f0g2z5d6n0b-np-delete-main-72qcj-6f66h to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z3q2f0g2z5d6n0b-np-delete-main-72qcj-6f66h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-9ngsk5k5bt5f--managed/providers/Microsoft.Compute/virtualMachines/z3q2f0g2z5d6n0b-np-delete-main-72qcj-6f66h to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dnr4nbhwmf28\" to finish creating: GET https://rp"}},{"row_id":"4fbd03a985ff188c193d13909c80b37024d29d2186902f7286502536ecdb8f54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103424207261011968","occurred_at":"2026-09-25T09:59:45Z","failed_at":"alert","signature_id":"d9de05c180e826fdf349a0c3e79eeeb21bd08be7c8f2912f4e152b0c6dfd4d1b","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T11:19:10Z\n Ended: 2026-09-25T11:39:12Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1011968-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1011968-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103423928134275072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T09:58:38Z","started_at":"2026-09-25T09:58:38Z","completed_at":"2026-09-25T12:27:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T12:36:19.334332581Z","timing_metadata_checked_at":"2026-09-25T12:36:19.334332581Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T12:36:19.337062473Z","region_metadata_checked_at":"2026-09-25T12:36:19.337062473Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"aafbc1e7ed1c1316981a1b0ba5a872819c06c6fe71972c92dd2765e189564cff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103423928134275072","occurred_at":"2026-09-25T09:58:38Z","failed_at":"alert","signature_id":"b16f25149b765227befbdd2221796f62e8724e78f44f61d5081a1d60dcec87db","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T11:13:21Z\n Ended: 2026-09-25T11:45:22Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4275072-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4275072-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"d29591884b88571ba7d5c92c335d419781eb64b6de212c7c00645a92dfd08c8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103423928134275072","occurred_at":"2026-09-25T09:58:38Z","failed_at":"alert","signature_id":"4126cd70f9e258007e4e457356ba9c9a093cd31c782620f814f33f5d61f037f4","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T11:19:57Z\n Ended: 2026-09-25T11:33:55Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4275072-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-09-25T11:19:57Z\n Ended: 2026-09-25T11:33:55Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4275072-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n State: Resolved\n Started: 2026-09-25T11:21:56Z\n Ended: 2026-09-25T11:31:55Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4275072-svc\", component=\"backend\", name=\"clusterclusterserviceupdatedispatch\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103423896454696960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"b938b99cc7c5a09bd230dfa6d470fb9e6a8d6e0a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-25T09:58:31Z","started_at":"2026-09-25T09:58:31Z","completed_at":"2026-09-25T12:24:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T12:36:19.153993355Z","timing_metadata_checked_at":"2026-09-25T12:36:19.153993355Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T12:36:19.156929946Z","region_metadata_checked_at":"2026-09-25T12:36:19.156929946Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103423876124905472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7118,"pr_state":"closed","pr_sha":"179ae75a7e3eff02b934146a76b63b8d8d485193","final_merged_sha":"f23942d30dc47ec7edeebb24ec30f183a59ea2fd","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-25T09:58:26Z","started_at":"2026-09-25T09:58:26Z","completed_at":"2026-09-25T12:24:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T12:36:19.499147857Z","timing_metadata_checked_at":"2026-09-25T12:36:19.499147857Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T12:36:19.501683949Z","region_metadata_checked_at":"2026-09-25T12:36:19.501683949Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103423873373442048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6503,"pr_state":"closed","pr_sha":"a0bf660bfdf32222374a82406e0907cae4b53869","final_merged_sha":"a0bf660bfdf32222374a82406e0907cae4b53869","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T09:58:25Z","started_at":"2026-09-25T09:58:25Z","completed_at":"2026-09-25T12:19:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T12:36:18.7310358Z","timing_metadata_checked_at":"2026-09-25T12:36:18.7310358Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T12:36:18.734814288Z","region_metadata_checked_at":"2026-09-25T12:36:18.734814288Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7150/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103423858945036288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7150,"pr_state":"closed","pr_sha":"5fdb2bd77b96d67d094d207e504542b274717b8f","final_merged_sha":"5fdb2bd77b96d67d094d207e504542b274717b8f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T09:58:22Z","started_at":"2026-09-25T09:58:22Z","completed_at":"2026-09-25T12:19:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T12:36:19.656736756Z","timing_metadata_checked_at":"2026-09-25T12:36:19.656736756Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T12:36:19.659546747Z","region_metadata_checked_at":"2026-09-25T12:36:19.659546747Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103423855413432320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"54fa83b6be2bda741a3e467491a8b8caf8a2dbe3","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T09:58:21Z","started_at":"2026-09-25T09:58:21Z","completed_at":"2026-09-25T12:25:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T12:36:18.870535556Z","timing_metadata_checked_at":"2026-09-25T12:36:18.870535556Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T12:36:18.874024945Z","region_metadata_checked_at":"2026-09-25T12:36:18.874024945Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"e295bd2d5bb15fe641031cc6fb1ee4d64a99027c3dc036c8613e89f3786fe136","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103423855413432320","occurred_at":"2026-09-25T09:58:21Z","failed_at":"alert","signature_id":"9c9e2ad911e851ab394d8666370f122b71b040daae5e9bd186a4b2a374c792dc","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T11:34:35Z\n Ended: 2026-09-25T11:41:37Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3432320-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3432320-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103419684916301824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6226,"pr_state":"closed","pr_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","final_merged_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T09:41:47Z","started_at":"2026-09-25T09:41:47Z","completed_at":"2026-09-25T12:00:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T12:14:42.740200954Z","timing_metadata_checked_at":"2026-09-25T12:14:42.740200954Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T12:14:42.744489741Z","region_metadata_checked_at":"2026-09-25T12:14:42.744489741Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103419069259583488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"3cae25cd3a41cc5575bd878ac507316389d59541","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T09:39:20Z","started_at":"2026-09-25T09:39:20Z","completed_at":"2026-09-25T11:58:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T12:14:43.14747764Z","timing_metadata_checked_at":"2026-09-25T12:14:43.14747764Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T12:14:43.150329731Z","region_metadata_checked_at":"2026-09-25T12:14:43.150329731Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"acd8fe329a391369d2658043f3dc7a3a83b929f2fa98d8496b087157ad46c64e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103419069259583488","occurred_at":"2026-09-25T09:39:20Z","failed_at":"alert","signature_id":"d3dbde3657783287fac24beab9467fbbf74dc1c5cfa00e1645a47049ac447877","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T10:43:07Z\n Ended: 2026-09-25T10:56:06Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9583488-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-09-25T10:44:05Z\n Ended: 2026-09-25T10:54:07Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9583488-svc\", component=\"backend\", name=\"clusterclusterserviceupdatedispatch\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103419021729730560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7110,"pr_state":"closed","pr_sha":"ce66d7285ab6bfc8f735b0e7c24632e94158eb25","final_merged_sha":"ce66d7285ab6bfc8f735b0e7c24632e94158eb25","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T09:39:09Z","started_at":"2026-09-25T09:39:09Z","completed_at":"2026-09-25T11:51:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T12:04:18.880148412Z","timing_metadata_checked_at":"2026-09-25T12:04:18.880148412Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T12:04:18.886358095Z","region_metadata_checked_at":"2026-09-25T12:04:18.886358095Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103413943887204352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7165,"pr_state":"open","pr_sha":"4ada4bfca590fd84b0b6be17befd2127f1df09bf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T09:18:58Z","started_at":"2026-09-25T09:18:58Z","completed_at":"2026-09-25T11:28:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T11:43:33.477140608Z","timing_metadata_checked_at":"2026-09-25T11:43:33.477140608Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T11:43:33.480297998Z","region_metadata_checked_at":"2026-09-25T11:43:33.480297998Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"c814636a7e5fa4434ca2b180699cc1cde1b6be6f88ee7e5a7a86b28ebedbec2b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103413943887204352","occurred_at":"2026-09-25T09:18:58Z","failed_at":"alert","signature_id":"726c0aa747c8650a3d621f95eb1b0811a06db840cb471b91ecb3ebd116e75b46","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T10:37:54Z\n Ended: 2026-09-25T10:43:57Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7204352-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7204352-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103412592713142272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"1fe1ab1561e20eccee307463926f02c7720fe0ae","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T09:13:36Z","started_at":"2026-09-25T09:13:36Z","completed_at":"2026-09-25T11:27:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T11:43:31.954298549Z","timing_metadata_checked_at":"2026-09-25T11:43:31.954298549Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T11:43:31.957714439Z","region_metadata_checked_at":"2026-09-25T11:43:31.957714439Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"39314e491e0352cfdacd080374719bee5b3d19bc1f3016b71bde5e9f48f58881","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103412592713142272","occurred_at":"2026-09-25T09:13:36Z","failed_at":"alert","signature_id":"8e8218e7a5915087525d6f8f3a88d4f7f906cb2241d2287308990610b3c657ab","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T10:31:40Z\n Ended: 2026-09-25T10:37:41Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3142272-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3142272-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103411413878837248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7164,"pr_state":"closed","pr_sha":"653ff4451ef4e6d7031565b7f3e25b8a6be9a485","final_merged_sha":"653ff4451ef4e6d7031565b7f3e25b8a6be9a485","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T09:08:54Z","started_at":"2026-09-25T09:08:54Z","completed_at":"2026-09-25T09:39:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T09:59:34.817188256Z","timing_metadata_checked_at":"2026-09-25T09:59:34.817188256Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T09:59:34.820487746Z","region_metadata_checked_at":"2026-09-25T09:59:34.820487746Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e6473ed6b1789d63651abe90207cce3c236766360a4a6f22b67d724cc2a2694d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103411413878837248","occurred_at":"2026-09-25T09:08:54Z","failed_at":"provision","signature_id":"21658b9b3d6cf9bf31715ba32c333953291d0d0fe4a6e4e41252f24603e05dff","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T09:33:51.597Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description=\"Step swift-vnet\\n Kind: Shell\\n Command: ./swift-vnet.sh\\n\"\ntime=2026-09-25T09:33:52.825Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: Launching container group swift-vnet-aks-net as globalMSI...\\nERROR: Please run 'az login' to setup account.\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"db15661ea057fcdb86696aa934e0ebb9321ec309068c024420faf52e725ffb06","failure_pattern":"ERROR: Please run 'az login' to setup account.","search_query":"ERROR: Please run 'az login' to setup account."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["db15661ea057fcdb86696aa934e0ebb9321ec309068c024420faf52e725ffb06"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103411013251502080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7071,"pr_state":"open","pr_sha":"6ed14b07bacae7ab466f489ecaf3984a528fce2a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T09:07:19Z","started_at":"2026-09-25T09:07:19Z","completed_at":"2026-09-25T11:26:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T11:43:32.428885103Z","timing_metadata_checked_at":"2026-09-25T11:43:32.428885103Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T11:43:32.431879994Z","region_metadata_checked_at":"2026-09-25T11:43:32.431879994Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"c934c3a1b0982c74634e08b29f573d83363502c112ab59adf7252574a0a58817","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103411013251502080","occurred_at":"2026-09-25T09:07:19Z","failed_at":"alert","signature_id":"1951379324799428f6c90ebb62ba30f9313ee199425535b87b2aa1e4bd2ab1eb","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T10:32:38Z\n Ended: 2026-09-25T10:38:35Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1502080-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1502080-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103408234294415360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7157,"pr_state":"closed","pr_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","final_merged_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T08:56:17Z","started_at":"2026-09-25T08:56:17Z","completed_at":"2026-09-25T11:27:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T11:43:32.776456443Z","timing_metadata_checked_at":"2026-09-25T11:43:32.776456443Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T11:43:32.779055536Z","region_metadata_checked_at":"2026-09-25T11:43:32.779055536Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"9b7759a3cd92546c5eb69e4275f0301f1ff0d4b41eb7d0ed59871bd3043d0375","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103408234294415360","occurred_at":"2026-09-25T08:56:17Z","failed_at":"alert","signature_id":"15a9e409aaf5d9e0579bf269fd83ede9ed5d3edd62cf633152f5854e716b53b1","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T10:32:52Z\n Ended: 2026-09-25T10:38:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4415360-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4415360-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103404569546461184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"open","pr_sha":"af7b58b5ec2512b233229edc9168efc02d636df0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T08:41:42Z","started_at":"2026-09-25T08:41:42Z","completed_at":"2026-09-25T10:52:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T11:12:41.607558918Z","timing_metadata_checked_at":"2026-09-25T11:12:41.607558918Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T11:12:41.613779998Z","region_metadata_checked_at":"2026-09-25T11:12:41.613779998Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"5d93f3c2c02d575b6e41fe9abcb817e15a3ee28018a4face947f2fb6d0039204","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103404569546461184","occurred_at":"2026-09-25T08:41:42Z","failed_at":"alert","signature_id":"50d133ff46da6182299e88907232ea570e9476c64cd6c905b70efc73336787e6","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T10:21:49Z\n Ended: 2026-09-25T10:27:50Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6461184-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j6461184-svc).\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-25T10:32:50Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6461184-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/providers/microsoft.redhatopenshift/locations/{location}/hcpoperationstatuses/{operationid}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/providers/microsoft.redhatopenshift/locations/{location}/hcpoperationstatuses/{operationid} exceeded 1 second over the past 30 minutes (cluster ci01-j6461184-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103400580599779328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"closed","pr_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","final_merged_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T08:25:52Z","started_at":"2026-09-25T08:25:52Z","completed_at":"2026-09-25T10:47:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T11:02:09.911258193Z","timing_metadata_checked_at":"2026-09-25T11:02:09.911258193Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T11:02:09.914656982Z","region_metadata_checked_at":"2026-09-25T11:02:09.914656982Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103397005047894016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7074,"pr_state":"open","pr_sha":"96d277dfa013bc1151443f1a591a57442af16c82","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T08:11:39Z","started_at":"2026-09-25T08:11:39Z","completed_at":"2026-09-25T10:29:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T10:41:05.854057573Z","timing_metadata_checked_at":"2026-09-25T10:41:05.854057573Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T10:41:05.856958664Z","region_metadata_checked_at":"2026-09-25T10:41:05.856958664Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"281bf0efc6cb2a7569944e36c4d7cbdd3bb841ceb8db2c20c8d922d5269ab42c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103397005047894016","occurred_at":"2026-09-25T08:11:39Z","failed_at":"alert","signature_id":"b6e3b81994212579ee43950ada16381b253fd6d13b3af77560b9b2fe98d3254e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T09:37:56Z\n Ended: 2026-09-25T09:43:51Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7894016-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7894016-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103396091608174592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T08:08:02Z","started_at":"2026-09-25T08:08:02Z","completed_at":"2026-09-25T10:23:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T10:41:05.653382716Z","timing_metadata_checked_at":"2026-09-25T10:41:05.653382716Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T10:41:05.656549606Z","region_metadata_checked_at":"2026-09-25T10:41:05.656549606Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1059e414e1a77f05c8def0166d5b7df91f299b5d6f713fd5d757c9755596a7c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103396091608174592","occurred_at":"2026-09-25T08:08:02Z","failed_at":"alert","signature_id":"971c1ccdcbd740d38dcc070a46e71996dd9726524b859ca87241878c2670d18c","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 20 time(s) (1 unknown, 19 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T09:10:28Z\n Ended: 2026-09-25T09:24:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8174592-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t1uirplqa1ftvrrcls451df8h5ktodf-cilium-cluster\", pod=\"router-6fd8bbf44f-qpddn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t1uirplqa1ftvrrcls451df8h5ktodf-cilium-cluster/router-6fd8bbf44f-qpddn has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"574d85b89affc3aa8d88bf29efa3cea35db17d1bbcbe0c90bc94063ee68f783c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103396091608174592","occurred_at":"2026-09-25T08:08:02Z","failed_at":"e2e","signature_id":"89cda5b6732c3ef010bea1f11ee4a5639cba21b5c854e052517ec53280975d30","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a4b500\u003e: \n failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a4b500\u003e: \n failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","failure_pattern":"ERROR CODE: 429; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: 429"}},{"row_id":"f43e04f2b20339d2d0fe3d5e2bbfda65ba401f9981bc37b7e9b3218d7af7ac38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103396091608174592","occurred_at":"2026-09-25T08:08:02Z","failed_at":"e2e","signature_id":"145d4ba5cffaac551d0281b7858ed436f7d626d510091524d5c39f50edc9ea43","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0026c0630\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-8b7lh78zv2w6\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-8b7lh78zv2w6, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0026c0630\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-8b7lh78zv2w6\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-8b7lh78zv2w6, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-w8sl5z8nrnnt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-w8sl5z8nrnnt, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103381642918760448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6687,"pr_state":"closed","pr_sha":"969fb59d00e72634fd03a9f90aa105a57f5aef4b","final_merged_sha":"969fb59d00e72634fd03a9f90aa105a57f5aef4b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T07:10:37Z","started_at":"2026-09-25T07:10:37Z","completed_at":"2026-09-25T09:19:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T09:38:14.701295405Z","timing_metadata_checked_at":"2026-09-25T09:38:14.701295405Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T09:38:14.704723195Z","region_metadata_checked_at":"2026-09-25T09:38:14.704723195Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103381081762828288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7074,"pr_state":"open","pr_sha":"96d277dfa013bc1151443f1a591a57442af16c82","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T07:08:23Z","started_at":"2026-09-25T07:08:23Z","completed_at":"2026-09-25T08:10:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T08:18:46.851295164Z","timing_metadata_checked_at":"2026-09-25T08:18:46.851295164Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T08:18:46.854222554Z","region_metadata_checked_at":"2026-09-25T08:18:46.854222554Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9af151799eda9109afa7f9585c31dd0b4762b21a2f05278cb481f91e70527c63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103381081762828288","occurred_at":"2026-09-25T07:08:23Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T07:34:22.688Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T07:34:24.657Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T07:34:25.413Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f9ded26ddc2f02a1d18fc804a27100f131aaebdd4730b75ce8e4252688ca94ad portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2828288%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff9ded26ddc2f02a1d18fc804a27100f131aaebdd4730b75ce8e4252688ca94ad\ntime=2026-09-25T07:36:56.106Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2828288/providers/Microsoft.Resources/deployments/f9ded26ddc2f02a1d18fc804a27100f131aaebdd4730b75ce8e4252688ca94ad/operationStatuses/08584112852205000524\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2828288/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2828288/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j2828288\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report ecacede2-e1f1-47b0-b03b-880ed0c8be61:9/25/2026 7:35:32 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T07:38:56.110Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T07:38:56.799Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T07:38:57.742Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e8a123abb52fb9b77644b6bb81f99052d395c9a86871b6051c52d46544bd7183 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2828288%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fe8a123abb52fb9b77644b6bb81f99052d395c9a86871b6051c52d46544bd7183\ntime=2026-09-25T07:55:32.210Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2828288/providers/Microsoft.Resources/deployments/e8a123abb52fb9b77644b6bb81f99052d395c9a86871b6051c52d46544bd7183/operationStatuses/08584112849482471420\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2828288/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2828288/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j2828288\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 94439e84-a993-4829-887d-7df8585944a7:9/25/2026 7:54:50 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T07:57:32.210Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T07:57:32.860Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T07:57:34.073Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=7b536fa82ce524300844a3f8a900c00ce0f29817f25e5d80c33937a4b6563a6c portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2828288%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F7b536fa82ce524300844a3f8a900c00ce0f29817f25e5d80c33937a4b6563a6c\ntime=2026-09-25T08:04:22.408Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103381037483560960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7146,"pr_state":"closed","pr_sha":"582984cb8d0a3a7745b66ed21bdf13dd9d4aead4","final_merged_sha":"34a27988a2da6ca8b624fbab48af075e636b7eb6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T07:08:12Z","started_at":"2026-09-25T07:08:12Z","completed_at":"2026-09-25T07:38:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T07:47:44.93066757Z","timing_metadata_checked_at":"2026-09-25T07:47:44.93066757Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T07:47:44.933118562Z","region_metadata_checked_at":"2026-09-25T07:47:44.933118562Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b8d054959d5f140f50fb174e96007731da865434f055437cf7958502c9b43d4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103381037483560960","occurred_at":"2026-09-25T07:08:12Z","failed_at":"provision","signature_id":"0d5e324b1f12b05a7f184d994200282ceecef7225105b3e1d8933bd3491b399c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/maestroMqttDataConnection","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T07:28:19.460Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n Kind: ARM\\n Template: templates/maestro-mqtt-data-connection.bicep\\n Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-25T07:28:19.869Z level=DEBUG+2 msg=\"Waiting for existing deployment to complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection deployment=67437eb973c49795934fc79733f3a01017eeb2994048cb050f5f7bf549795e78\ntime=2026-09-25T07:28:19.869Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to poll previously-existing deployment: pre-existing deployment failed: {\\\"code\\\":\\\"DeploymentFailed\\\",\\\"details\\\":[{\\\"code\\\":\\\"EventHubValidationErrorFound\\\",\\\"message\\\":\\\"[BadRequest] Validation Errors found: Table does not exist\\\"}],\\\"message\\\":\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\"}\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7772a8fa7766108b00c98006e84756f50f97d8f9c7242d532de03696ce3d923c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code EventHubValidationErrorFound; detail message [BadRequest] Validation Errors found: Table does not exist","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7772a8fa7766108b00c98006e84756f50f97d8f9c7242d532de03696ce3d923c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7150/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103380979837046784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7150,"pr_state":"closed","pr_sha":"5fdb2bd77b96d67d094d207e504542b274717b8f","final_merged_sha":"5fdb2bd77b96d67d094d207e504542b274717b8f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T07:07:59Z","started_at":"2026-09-25T07:07:59Z","completed_at":"2026-09-25T08:09:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T08:18:47.296701048Z","timing_metadata_checked_at":"2026-09-25T08:18:47.296701048Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T08:18:47.29910514Z","region_metadata_checked_at":"2026-09-25T08:18:47.29910514Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1c0de37d9ebab49d0b8fb03aeac030e5f46f682203d13c1902207b4e00fa2f3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7150/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103380979837046784","occurred_at":"2026-09-25T07:07:59Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T07:34:31.218Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T07:34:32.996Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T07:34:33.642Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=a9aff744a2cdd92874d7db6d918f25d81ca7457483a3a6907d839e321dc99f0f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7046784%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fa9aff744a2cdd92874d7db6d918f25d81ca7457483a3a6907d839e321dc99f0f\ntime=2026-09-25T07:37:04.387Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7046784/providers/Microsoft.Resources/deployments/a9aff744a2cdd92874d7db6d918f25d81ca7457483a3a6907d839e321dc99f0f/operationStatuses/08584112852122163443\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7046784/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7046784/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j7046784\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 9b929ab0-5c2e-49dd-bbb2-9ce2e31da1d6:9/25/2026 7:35:45 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T07:39:04.396Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T07:39:05.205Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T07:39:05.853Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=0207d870a10f285bc5d88ad01d70dcbec98e47333e083f707a356e8195d86281 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7046784%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F0207d870a10f285bc5d88ad01d70dcbec98e47333e083f707a356e8195d86281\ntime=2026-09-25T07:55:09.875Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7046784/providers/Microsoft.Resources/deployments/0207d870a10f285bc5d88ad01d70dcbec98e47333e083f707a356e8195d86281/operationStatuses/08584112849400508889\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7046784/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7046784/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j7046784\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 16a0f773-9b58-4472-8c4f-1a09ae6a01ad:9/25/2026 7:54:45 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T07:57:09.880Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T07:57:10.721Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T07:57:12.170Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=93084b613ff5ea8055dea9e891463c115285c4a6e976c476d8ff18b04ab4a30f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7046784%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F93084b613ff5ea8055dea9e891463c115285c4a6e976c476d8ff18b04ab4a30f\ntime=2026-09-25T08:04:30.925Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103373632955748352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"89d7db87442ead3f3e8fcb09b59a5aaf74769ffd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-25T06:38:47Z","started_at":"2026-09-25T06:38:47Z","completed_at":"2026-09-25T08:49:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T08:59:52.25111823Z","timing_metadata_checked_at":"2026-09-25T08:59:52.25111823Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T08:59:52.255417319Z","region_metadata_checked_at":"2026-09-25T08:59:52.255417319Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103369630306275328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7123,"pr_state":"closed","pr_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","final_merged_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T06:22:53Z","started_at":"2026-09-25T06:22:53Z","completed_at":"2026-09-25T07:36:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T07:47:44.804779975Z","timing_metadata_checked_at":"2026-09-25T07:47:44.804779975Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T07:47:44.807198367Z","region_metadata_checked_at":"2026-09-25T07:47:44.807198367Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d924b843e3a62ca0bd1d796c530a1b3996efde423a98ebb0e45e8a538258be08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103369630306275328","occurred_at":"2026-09-25T06:22:53Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T06:45:27.884Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T06:45:29.582Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T06:45:30.552Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=792667e0833f10bd66e53137e529fbb915007ee36bf88d664c73749427752b01 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6275328%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F792667e0833f10bd66e53137e529fbb915007ee36bf88d664c73749427752b01\ntime=2026-09-25T06:48:01.276Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6275328/providers/Microsoft.Resources/deployments/792667e0833f10bd66e53137e529fbb915007ee36bf88d664c73749427752b01/operationStatuses/08584112881555510636\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6275328/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6275328/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6275328\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report a63f0b8e-33cd-496a-b32d-b6f7bac8df79:9/25/2026 6:47:10 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T06:50:01.286Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T06:50:01.864Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T06:50:02.669Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=4de62d3c0202189802245d6397e5da77cc4c50b779ec96e3edd98b8c97b12b9f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6275328%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4de62d3c0202189802245d6397e5da77cc4c50b779ec96e3edd98b8c97b12b9f\ntime=2026-09-25T07:06:06.376Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6275328/providers/Microsoft.Resources/deployments/4de62d3c0202189802245d6397e5da77cc4c50b779ec96e3edd98b8c97b12b9f/operationStatuses/08584112878833453540\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6275328/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6275328/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6275328\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 60c77343-ca84-4273-9e67-f47965c02905:9/25/2026 7:05:54 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T07:08:06.380Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T07:08:07.065Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T07:08:07.806Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=22d9ef212af7c5cfa935b24d7e825924c91cea0072f3d8e6a5e43e6e27a479f2 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6275328%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F22d9ef212af7c5cfa935b24d7e825924c91cea0072f3d8e6a5e43e6e27a479f2\ntime=2026-09-25T07:15:27.634Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103369316203237376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7135,"pr_state":"closed","pr_sha":"78d2e7cdedac95816c4452d91dfcae35ac93f66a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T06:21:38Z","started_at":"2026-09-25T06:21:38Z","completed_at":"2026-09-25T07:27:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T07:36:57.539465854Z","timing_metadata_checked_at":"2026-09-25T07:36:57.539465854Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T07:36:57.542034047Z","region_metadata_checked_at":"2026-09-25T07:36:57.542034047Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"357956e6c01c54529f1bc02b46890d1b50e6712f2a27a26fcfa284f950ae9cbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103369316203237376","occurred_at":"2026-09-25T06:21:38Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T06:44:48.252Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T06:44:49.723Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T06:44:50.523Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c3acf1ef94ba04ce37f83b4d82d75e36190420d4acc62209bf0c93f6eb128dca portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3237376%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc3acf1ef94ba04ce37f83b4d82d75e36190420d4acc62209bf0c93f6eb128dca\ntime=2026-09-25T06:47:21.372Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3237376/providers/Microsoft.Resources/deployments/c3acf1ef94ba04ce37f83b4d82d75e36190420d4acc62209bf0c93f6eb128dca/operationStatuses/08584112881953675870\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3237376/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3237376/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3237376\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report eadbdd90-d126-410b-99fa-d6c9102ff8d6:9/25/2026 6:46:30 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T06:49:21.373Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T06:49:22.299Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T06:49:23.075Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=ec440c3d7acf83ae5d53306b7eea966201ea26ad4bb986792c1d75b6a9c10b67 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3237376%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fec440c3d7acf83ae5d53306b7eea966201ea26ad4bb986792c1d75b6a9c10b67\ntime=2026-09-25T07:05:27.416Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3237376/providers/Microsoft.Resources/deployments/ec440c3d7acf83ae5d53306b7eea966201ea26ad4bb986792c1d75b6a9c10b67/operationStatuses/08584112879228428091\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3237376/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3237376/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3237376\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 37b6f03b-a2ac-4c27-88ab-5bb109a62d28:9/25/2026 7:05:12 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T07:07:27.417Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T07:07:28.208Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T07:07:29.170Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=92df036016ec7119555e4b184c80d201c6b4cb3d779d60e087ed965b80ccb391 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3237376%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F92df036016ec7119555e4b184c80d201c6b4cb3d779d60e087ed965b80ccb391\ntime=2026-09-25T07:14:47.849Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103349066208382976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7148,"pr_state":"closed","pr_sha":"c3e1a8ef598ecb7f85e0ae9a62013997e2cf959c","final_merged_sha":"c3e1a8ef598ecb7f85e0ae9a62013997e2cf959c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T05:00:56Z","started_at":"2026-09-25T05:00:56Z","completed_at":"2026-09-25T07:26:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T07:36:57.68171666Z","timing_metadata_checked_at":"2026-09-25T07:36:57.68171666Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T07:36:57.684296553Z","region_metadata_checked_at":"2026-09-25T07:36:57.684296553Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103346399469899776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7157,"pr_state":"closed","pr_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","final_merged_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T04:50:34Z","started_at":"2026-09-25T04:50:34Z","completed_at":"2026-09-25T07:13:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T07:26:53.776676178Z","timing_metadata_checked_at":"2026-09-25T07:26:53.776676178Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T07:26:53.780187167Z","region_metadata_checked_at":"2026-09-25T07:26:53.780187167Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"42ef463ac35a8cf87f2f10374150863786a243967b4c20e431672d7a0df7bda2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103346399469899776","occurred_at":"2026-09-25T04:50:34Z","failed_at":"alert","signature_id":"4b57206254fb7de2fa5b705bd0848fd62b6ad1452a8b7a4204cadac985929aef","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T05:59:09Z\n Ended: 2026-09-25T06:27:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9899776-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t1rpl9s1tcrsg2cgvm00adhvd6vdptj-e9w5w9v2w8j5n9x\", pod=\"router-78ccb5c69b-67j5d\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t1rpl9s1tcrsg2cgvm00adhvd6vdptj-e9w5w9v2w8j5n9x/router-78ccb5c69b-67j5d has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-25T06:00:11Z\n Ended: 2026-09-25T06:27:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9899776-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t1rpj5sqn7hprctp4odc01ldkll7cd0-private-kas\", pod=\"router-765df84757-2ds67\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t1rpj5sqn7hprctp4odc01ldkll7cd0-private-kas/router-765df84757-2ds67 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"7077aee969cfdf638e3946a7679702d0b882bec9877dc25d7b4a40d981cc954c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103346399469899776","occurred_at":"2026-09-25T04:50:34Z","failed_at":"e2e","signature_id":"42a26a00c059eba85e0180aa4e4f83ccc2a993aadc3308bd54f4a077fbc7477d","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1aa00\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pgng9vz6j8w8\" to finish creating: GET https://rp.j9899776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a614d325-8937-4e4c-a91f-47b2564f0327\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a614d325-8937-4e4c-a91f-47b2564f0327\",\n \"name\": \"a614d325-8937-4e4c-a91f-47b2564f0327\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T05:48:11.218096649Z\",\n \"endTime\": \"2026-09-25T06:17:14.62086737Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m30s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1aa00\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pgng9vz6j8w8\" to finish creating: GET https://rp.j9899776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a614d325-8937-4e4c-a91f-47b2564f0327\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a614d325-8937-4e4c-a91f-47b2564f0327\",\n \"name\": \"a614d325-8937-4e4c-a91f-47b2564f0327\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T05:48:11.218096649Z\",\n \"endTime\": \"2026-09-25T06:17:14.62086737Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m30s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8bd8c54766893120df59867b74aa4b14e5a53af3cf15db456c35fdb858012db9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103346399469899776","occurred_at":"2026-09-25T04:50:34Z","failed_at":"e2e","signature_id":"27a55be51755e80b2d85191d7869a0f17115a878504d6f559e3291475acc6d1a","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--6ghnhlc94h8f/cluster-hs-pre-5wm67n should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f25a0\u003e: \n failed to create HCP cluster cluster-hs-pre-5wm67n: failed waiting for cluster=\"cluster-hs-pre-5wm67n\" in resourcegroup=\"rg-hs-pre--6ghnhlc94h8f\" to finish creating: GET https://rp.j9899776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46dab79f-2360-4680-914d-589d803cc911\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46dab79f-2360-4680-914d-589d803cc911\",\n \"name\": \"46dab79f-2360-4680-914d-589d803cc911\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T05:47:00.77129988Z\",\n \"endTime\": \"2026-09-25T06:16:03.97431147Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--6ghnhlc94h8f/cluster-hs-pre-5wm67n should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f25a0\u003e: \n failed to create HCP cluster cluster-hs-pre-5wm67n: failed waiting for cluster=\"cluster-hs-pre-5wm67n\" in resourcegroup=\"rg-hs-pre--6ghnhlc94h8f\" to finish creating: GET https://rp.j9899776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46dab79f-2360-4680-914d-589d803cc911\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46dab79f-2360-4680-914d-589d803cc911\",\n \"name\": \"46dab79f-2360-4680-914d-589d803cc911\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T05:47:00.77129988Z\",\n \"endTime\": \"2026-09-25T06:16:03.97431147Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fc93fe321ea2615952ad44709fbb1185dee8071206d61b278458646b673d567d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103346399469899776","occurred_at":"2026-09-25T04:50:34Z","failed_at":"e2e","signature_id":"5f77dd910e6de408d94f4725f867f2110f5e4dffba593f9da2e73cbb55e65e8c","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b22160\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-cxw6lsrpfgzt\" to finish creating: GET https://rp.j9899776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a684a391-ad01-4109-a5eb-484adf2cf82c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a684a391-ad01-4109-a5eb-484adf2cf82c\",\n \"name\": \"a684a391-ad01-4109-a5eb-484adf2cf82c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T05:46:50.496521788Z\",\n \"endTime\": \"2026-09-25T06:15:53.817065387Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.15 is Partial (want Completed), started 22m23s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b22160\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-cxw6lsrpfgzt\" to finish creating: GET https://rp.j9899776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a684a391-ad01-4109-a5eb-484adf2cf82c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a684a391-ad01-4109-a5eb-484adf2cf82c\",\n \"name\": \"a684a391-ad01-4109-a5eb-484adf2cf82c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T05:46:50.496521788Z\",\n \"endTime\": \"2026-09-25T06:15:53.817065387Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.15 is Partial (want Completed), started 22m23s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7153/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103341305806458880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7153,"pr_state":"closed","pr_sha":"9520bc407b831edebb52b2748177fd2abb1be351","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T04:30:16Z","started_at":"2026-09-25T04:30:16Z","completed_at":"2026-09-25T04:46:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T05:01:08.750249405Z","timing_metadata_checked_at":"2026-09-25T05:01:08.750249405Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T05:01:08.753402496Z","region_metadata_checked_at":"2026-09-25T05:01:08.753402496Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c935976c34b711f3506241e7196b803975e79e655745b8bb4c0e52fc764298d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7153/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103341305806458880","occurred_at":"2026-09-25T04:30:16Z","failed_at":"provision","signature_id":"eb3a097ae9ea09f1b87574d34137bdc9344882caed893e38ceda6a45fc8e1166","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/maestroMqttDataConnection","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T04:41:01.572Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n Kind: ARM\\n Template: templates/maestro-mqtt-data-connection.bicep\\n Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-25T04:41:01.944Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection\ntime=2026-09-25T04:41:02.776Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection deployment=be82b2c4bc5e91c9fb05bb7a64319b66570f7844a6d32af17e112fe3dd39a4e7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-kusto-us%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fbe82b2c4bc5e91c9fb05bb7a64319b66570f7844a6d32af17e112fe3dd39a4e7\ntime=2026-09-25T04:41:32.894Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/be82b2c4bc5e91c9fb05bb7a64319b66570f7844a6d32af17e112fe3dd39a4e7/operationStatuses/08584112956230068016\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"EventHubValidationErrorFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"[BadRequest] Validation Errors found: Table does not exist\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7772a8fa7766108b00c98006e84756f50f97d8f9c7242d532de03696ce3d923c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code EventHubValidationErrorFound; detail message [BadRequest] Validation Errors found: Table does not exist","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7772a8fa7766108b00c98006e84756f50f97d8f9c7242d532de03696ce3d923c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103339957669072896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7065,"pr_state":"open","pr_sha":"015b6f8ee1e67cb89d64d0f0068341e677bbc121","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-25T04:24:58Z","started_at":"2026-09-25T04:24:58Z","completed_at":"2026-09-25T06:32:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T06:45:47.983380175Z","timing_metadata_checked_at":"2026-09-25T06:45:47.983380175Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T06:45:47.986672964Z","region_metadata_checked_at":"2026-09-25T06:45:47.986672964Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7156/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103336399548715008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7156,"pr_state":"closed","pr_sha":"76fcf7ba66f401f373d847f8cb0bc0eb74b157b4","final_merged_sha":"76fcf7ba66f401f373d847f8cb0bc0eb74b157b4","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T04:10:50Z","started_at":"2026-09-25T04:10:50Z","completed_at":"2026-09-25T05:04:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T05:11:18.509088547Z","timing_metadata_checked_at":"2026-09-25T05:11:18.509088547Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T05:11:18.511982737Z","region_metadata_checked_at":"2026-09-25T05:11:18.511982737Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"216fb88031ee9b865a95d749af43362f78d1d317feee1a6d3e8863c484541bd3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7156/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103336399548715008","occurred_at":"2026-09-25T04:10:50Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T04:27:47.092Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T04:27:48.756Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T04:27:49.522Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=b0fec80bd91a4428f86153eba599bc5e89b51b50a6cd92111176663d87fc5684 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8715008%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb0fec80bd91a4428f86153eba599bc5e89b51b50a6cd92111176663d87fc5684\ntime=2026-09-25T04:30:20.274Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8715008/providers/Microsoft.Resources/deployments/b0fec80bd91a4428f86153eba599bc5e89b51b50a6cd92111176663d87fc5684/operationStatuses/08584112964164335429\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8715008/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8715008/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8715008\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 6066dd0b-807f-4ab0-9ae2-101a204225e6:9/25/2026 4:29:04 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T04:32:20.276Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T04:32:21.007Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T04:32:22.179Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8528e829fe21798387418d9e980e31dc20d5a0d8608408f1d332e67bad13e149 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8715008%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8528e829fe21798387418d9e980e31dc20d5a0d8608408f1d332e67bad13e149\ntime=2026-09-25T04:48:26.877Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8715008/providers/Microsoft.Resources/deployments/8528e829fe21798387418d9e980e31dc20d5a0d8608408f1d332e67bad13e149/operationStatuses/08584112961439051679\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8715008/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8715008/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8715008\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report a6ac9074-5a28-4dd4-a0c7-4c07be75719c:9/25/2026 4:47:59 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T04:50:26.886Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T04:50:27.553Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T04:50:28.225Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=595d74afc833783a1aee61884cad125b799006289619a8b5ea0fa6a211c78cee portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8715008%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F595d74afc833783a1aee61884cad125b799006289619a8b5ea0fa6a211c78cee\ntime=2026-09-25T04:57:46.747Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7131/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103327844472983552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7131,"pr_state":"closed","pr_sha":"cb8c2640a77a75ae501a0a609a7518ef1807d76b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T03:36:50Z","started_at":"2026-09-25T03:36:50Z","completed_at":"2026-09-25T03:49:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T03:58:35.48441031Z","timing_metadata_checked_at":"2026-09-25T03:58:35.48441031Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-25T03:58:35.486909902Z","region_metadata_checked_at":"2026-09-25T05:01:08.460289914Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6c128baf5465cdb3f46b4d7000f19700eac0c35d0844404695fed8635385d9e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7131/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103327844472983552","occurred_at":"2026-09-25T03:36:50Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103327464351600640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7133,"pr_state":"open","pr_sha":"c2d1719d2c83ce4eb02f5e7803f798b5b01e417f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T03:35:15Z","started_at":"2026-09-25T03:35:15Z","completed_at":"2026-09-25T03:46:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T03:58:35.590985295Z","timing_metadata_checked_at":"2026-09-25T03:58:35.590985295Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-25T03:58:35.593446387Z","region_metadata_checked_at":"2026-09-25T05:01:08.560307335Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c92d904e271ab295cdb858c07356cd9fd080064d726cb1bc4729b17f6f6e42ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103327464351600640","occurred_at":"2026-09-25T03:35:15Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318302158753792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"b938b99cc7c5a09bd230dfa6d470fb9e6a8d6e0a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T02:58:55Z","started_at":"2026-09-25T02:58:55Z","completed_at":"2026-09-25T03:58:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T04:08:44.347322853Z","timing_metadata_checked_at":"2026-09-25T04:08:44.347322853Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T04:08:44.354660029Z","region_metadata_checked_at":"2026-09-25T04:08:44.354660029Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"57fb6ec0456d4d362c46dd15bec501b89ce1bd4f60174284c6517c8b68cab521","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318302158753792","occurred_at":"2026-09-25T02:58:55Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T03:24:12.964Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T03:24:14.582Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T03:24:15.455Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c8b164201d40631780f298d68a514b9f78677a96e9cb8704a0ebba3dade3a637 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8753792%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc8b164201d40631780f298d68a514b9f78677a96e9cb8704a0ebba3dade3a637\ntime=2026-09-25T03:26:46.245Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8753792/providers/Microsoft.Resources/deployments/c8b164201d40631780f298d68a514b9f78677a96e9cb8704a0ebba3dade3a637/operationStatuses/08584113002306312931\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8753792/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8753792/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8753792\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report d3c94bc2-9616-4360-b499-9ba1cb9249f9:9/25/2026 3:25:37 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T03:28:46.250Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T03:28:47.031Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T03:28:48.016Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=fb357a61bfe2f4469350a90ca0538e9f4b6b47928a23f22ea3053fd100d93646 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8753792%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ffb357a61bfe2f4469350a90ca0538e9f4b6b47928a23f22ea3053fd100d93646\ntime=2026-09-25T03:44:52.948Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8753792/providers/Microsoft.Resources/deployments/fb357a61bfe2f4469350a90ca0538e9f4b6b47928a23f22ea3053fd100d93646/operationStatuses/08584112999581574521\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8753792/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8753792/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8753792\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 4cc7b958-4dac-4f78-95f3-b1c53bb2e453:9/25/2026 3:44:09 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T03:46:52.949Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T03:46:53.764Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T03:46:55.097Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=0d6f91729539458b7743ee374377a666cd69454dfcd2490894b835a5b1c12c23 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8753792%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F0d6f91729539458b7743ee374377a666cd69454dfcd2490894b835a5b1c12c23\ntime=2026-09-25T03:54:12.659Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318277487857664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7135,"pr_state":"closed","pr_sha":"78d2e7cdedac95816c4452d91dfcae35ac93f66a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T02:58:49Z","started_at":"2026-09-25T02:58:49Z","completed_at":"2026-09-25T05:26:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T05:32:57.11421652Z","timing_metadata_checked_at":"2026-09-25T05:32:57.11421652Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T05:32:57.116759212Z","region_metadata_checked_at":"2026-09-25T05:32:57.116759212Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"3d5d300fdc224d6d7d5ec5b0065f98e96dfd9a852010d515d5494896cbbd3406","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318277487857664","occurred_at":"2026-09-25T02:58:49Z","failed_at":"e2e","signature_id":"ba0a7ce2e48b37c08aaa14a387af90804db2c47729b8040cba4bb165d499132f","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006377a0\u003e: \n failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-8lt5rxb4tbjz\" to finish creating: GET https://rp.j7857664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ed95f0cc-a570-4ff6-b5aa-553b0f2e32b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ed95f0cc-a570-4ff6-b5aa-553b0f2e32b6\",\n \"name\": \"ed95f0cc-a570-4ff6-b5aa-553b0f2e32b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T04:05:25.290345016Z\",\n \"endTime\": \"2026-09-25T04:24:25.319865016Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-6qhsv-d96qd: Failed: interfaces failed to create or update. err: failed to get existing resource ephemeral-osdisk-8lt5rxb4tbjz--managed/ephemeral-disk-ephemeral-np-6qhsv-d96qd-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-8lt5rxb4tbjz--managed/providers/Microsoft.Network/networkInterfaces/ephemeral-disk-ephemeral-np-6qhsv-d96qd-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '10' seconds after additional token... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006377a0\u003e: \n failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-8lt5rxb4tbjz\" to finish creating: GET https://rp.j7857664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ed95f0cc-a570-4ff6-b5aa-553b0f2e32b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ed95f0cc-a570-4ff6-b5aa-553b0f2e32b6\",\n \"name\": \"ed95f0cc-a570-4ff6-b5aa-553b0f2e32b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T04:05:25.290345016Z\",\n \"endTime\": \"2026-09-25T04:24:25.319865016Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-6qhsv-d96qd: Failed: interfaces failed to create or update. err: failed to get existing resource ephemeral-osdisk-8lt5rxb4tbjz--managed/ephemeral-disk-ephemeral-np-6qhsv-d96qd-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-8lt5rxb4tbjz--managed/providers/Microsoft.Network/networkInterfaces/ephemeral-disk-ephemeral-np-6qhsv-d96qd-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '10' seconds after additional token... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3d6035dc15a6de29702ad772c845059ee997267e8ce8729729aed4e2cee290dd","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional token... message truncated; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eb74f28dd229536da0eb17b70c1e7834c0ef679225bf2dcf23ff995c90d6c913","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318277487857664","occurred_at":"2026-09-25T02:58:49Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T04:10:59Z\n Ended: 2026-09-25T04:25:57Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7857664-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"f3c0c0a5aa0f98538ecb640a2b5aeedaf20ed241489af4e285d7149af38469cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318277487857664","occurred_at":"2026-09-25T02:58:49Z","failed_at":"e2e","signature_id":"c493dd3074bde168273260831e1897caadd72a13f32f14f010c37424a8cb9ea8","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa2920\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-x4kl262s2lph\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa2920\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-x4kl262s2lph\" to finish creating: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-x4kl262s2lph\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","3d6035dc15a6de29702ad772c845059ee997267e8ce8729729aed4e2cee290dd","d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318256310816768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6503,"pr_state":"closed","pr_sha":"a0bf660bfdf32222374a82406e0907cae4b53869","final_merged_sha":"a0bf660bfdf32222374a82406e0907cae4b53869","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T02:58:44Z","started_at":"2026-09-25T02:58:44Z","completed_at":"2026-09-25T03:59:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T04:08:44.114819406Z","timing_metadata_checked_at":"2026-09-25T04:08:44.114819406Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T04:08:44.118369094Z","region_metadata_checked_at":"2026-09-25T04:08:44.118369094Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5defe1cc6d4f24ffb463d2fc7965f38e42a51cc8a4cf7197b74a12c082f0b1c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318256310816768","occurred_at":"2026-09-25T02:58:44Z","failed_at":"provision","signature_id":"600bfb947207083afdd67c9ce6feb6937313b5ee52299c05181ebcb8242f4515","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/aro-hcp-exporter","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T03:43:40.223Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter description=\"Step aro-hcp-exporter\\n Kind: Helm\\n\"\ntime=2026-09-25T03:43:40.544Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter values=\"map[cacheTTL:60m certificateNames:[frontend-cert-ci01-j0816768 admin-api-cert-ci01-j0816768 sessiongate-cert-ci01-j0816768] clusterNameFilter:ci01-j0816768 clusterTypes:svc-cluster,mgmt-cluster collectionInterval:2m deployment:map[name:aro-hcp-exporter] enabledCollectors:service-tag-usage,kusto-logs-current,keyvault-certificates image:map[digest:sha256:742269f198294ce81d1f0d1f1fa3db44028559acb86d454e16d8997543a54aa4 registry:arohcpsvcdev.azurecr.io repository:ci-op-01nc500m/pipeline] keyvaultDNSSuffix:vault.azure.net keyvaultName:aro-hcp-dev-svc-kv kustoCluster:hcp-dev-us-2 kustoRegion:East US 2 listenAddress::8080 region:canadacentral resources:map[limits:map[memory:unlimited] requests:map[cpu:100m memory:128Mi]] service:map[name:aro-hcp-exporter] serviceAccount:map[create:true msiClientId:9cf3e005-99aa-4353-992d-fc05ca747f7f name:aro-hcp-exporter]]\"\ntime=2026-09-25T03:43:40.544Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-09-25T03:43:40.690Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter namespace=aro-hcp-exporter\ntime=2026-09-25T03:43:41.003Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter dryRun=false\ntime=2026-09-25T03:43:41.003Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-09-25T03:43:41.052Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-09-25T03:48:43.009Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter err=\"resource Deployment/aro-hcp-exporter/aro-hcp-exporter not ready. status: InProgress, message: Replicas: 0/1\\ncontext deadline exceeded\"\ntime=2026-09-25T03:48:43.009Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp-exporter/aro-hcp-exporter not ready. status: InProgress, message: Replicas: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"54c2b034583e4a5d24eb9d13c73dcfe6a50efc774d9a44838d433b2a32ba910a","failure_pattern":"failed to roll out Helm release: resource Deployment/aro-hcp-exporter/aro-hcp-exporter not ready. status: InProgress, message: Replicas: 0/1 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp-exporter/aro-hcp-exporter not ready. status: InProgress, message: Replicas: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["54c2b034583e4a5d24eb9d13c73dcfe6a50efc774d9a44838d433b2a32ba910a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7150/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318253316083712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7150,"pr_state":"closed","pr_sha":"7a563cc5f0275dd24c6edaf42691aeac6d6897a5","final_merged_sha":"5fdb2bd77b96d67d094d207e504542b274717b8f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T02:58:43Z","started_at":"2026-09-25T02:58:43Z","completed_at":"2026-09-25T05:16:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T05:22:00.888068876Z","timing_metadata_checked_at":"2026-09-25T05:22:00.888068876Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T05:22:00.890897867Z","region_metadata_checked_at":"2026-09-25T05:22:00.890897867Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"9cf37f79d0640b1afd949d086394e1a2ccc85ba9c922cbc7f215d5e0b2d7ac81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7150/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318253316083712","occurred_at":"2026-09-25T02:58:43Z","failed_at":"alert","signature_id":"0b5e495e1b35ed4266a96aacd072322525c0cf00dc1a7b76b0d2653c3c70a28a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T04:28:24Z\n Ended: 2026-09-25T04:34:27Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6083712-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6083712-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318216586563584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"54fa83b6be2bda741a3e467491a8b8caf8a2dbe3","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T02:58:35Z","started_at":"2026-09-25T02:58:35Z","completed_at":"2026-09-25T05:21:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T05:32:56.769993836Z","timing_metadata_checked_at":"2026-09-25T05:32:56.769993836Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T05:32:56.772648728Z","region_metadata_checked_at":"2026-09-25T05:32:56.772648728Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"94f1f7f84cbf618de1f96ea3226e332d921a0e0c7115c810da67ccc9c5060790","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318216586563584","occurred_at":"2026-09-25T02:58:35Z","failed_at":"alert","signature_id":"c148dd930f4f1038c64e5bda079b2babb2225218b71356ce91b62e5328d1c0a7","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T04:30:27Z\n Ended: 2026-09-25T04:36:28Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6563584-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6563584-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318217182154752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T02:58:35Z","started_at":"2026-09-25T02:58:35Z","completed_at":"2026-09-25T05:26:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T05:32:56.93859489Z","timing_metadata_checked_at":"2026-09-25T05:32:56.93859489Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T05:32:56.941385181Z","region_metadata_checked_at":"2026-09-25T05:32:56.941385181Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"99bd52259a1d85d8542e29779e8bc2d0e868c8f676dd985a6c3ae44b1c4ac675","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318217182154752","occurred_at":"2026-09-25T02:58:35Z","failed_at":"alert","signature_id":"52816a71ef2a03b49578fdbb7a916c3c123b4226a4221c05983979cb58643e9c","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T04:21:35Z\n Ended: 2026-09-25T04:28:34Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2154752-svc\", component=\"backend\", name=\"datadump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"9d5a30927f5e159e56cdead390e8e266ad4a91c93c4c1e926ab5f39a47aba895","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318217182154752","occurred_at":"2026-09-25T02:58:35Z","failed_at":"alert","signature_id":"f4e4b5d37bf82f3189cfb3aa38f15d4ff310e4e09323e5b109f9510546431a86","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T03:58:54Z\n Ended: 2026-09-25T04:35:01Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2154752-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2154752-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318212358705152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7146,"pr_state":"closed","pr_sha":"210214039c8925fdc446fab903f1f8d1a21e74ae","final_merged_sha":"34a27988a2da6ca8b624fbab48af075e636b7eb6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T02:58:34Z","started_at":"2026-09-25T02:58:34Z","completed_at":"2026-09-25T03:28:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T03:37:49.048780296Z","timing_metadata_checked_at":"2026-09-25T03:37:49.048780296Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T03:37:49.05578976Z","region_metadata_checked_at":"2026-09-25T03:37:49.05578976Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4076613f14e49434850da735fd5e783cc17c0d1799b33811a5af00a6c973967c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318212358705152","occurred_at":"2026-09-25T02:58:34Z","failed_at":"provision","signature_id":"8b0f84d3fd13e52b773c2d5762e6e413e0d4ff59cb390c1fe435499d6e4b2b92","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/maestroMqttDataConnection","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T03:23:26.924Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n Kind: ARM\\n Template: templates/maestro-mqtt-data-connection.bicep\\n Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-25T03:23:27.268Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection\ntime=2026-09-25T03:23:27.485Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/23db75d3c4a6f5fe32de7df35567f808137cc8acac2d401192c396cc43f1121e\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplate\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\n \\\"message\\\": \\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\n \\\"additionalInfo\\\": [\\n {\\n \\\"type\\\": \\\"TemplateViolation\\\",\\n \\\"info\\\": {\\n \\\"lineNumber\\\": 0,\\n \\\"linePosition\\\": 0,\\n \\\"path\\\": \\\"\\\"\\n }\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0905a6f0720326490c59375e0404554c58e1e0996cc236e2a705a956863de47f","failure_pattern":"ERROR CODE: InvalidTemplate; detail message Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'.","search_query":"ERROR CODE: InvalidTemplate"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0905a6f0720326490c59375e0404554c58e1e0996cc236e2a705a956863de47f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7149/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318186576318464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7149,"pr_state":"closed","pr_sha":"75f4b3d73fbc760e8b69df0db4067b8702fb2556","final_merged_sha":"75f4b3d73fbc760e8b69df0db4067b8702fb2556","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T02:58:28Z","started_at":"2026-09-25T02:58:28Z","completed_at":"2026-09-25T05:27:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T05:32:57.248762684Z","timing_metadata_checked_at":"2026-09-25T05:32:57.248762684Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T05:32:57.251435175Z","region_metadata_checked_at":"2026-09-25T05:32:57.251435175Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318164392644608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7123,"pr_state":"closed","pr_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","final_merged_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T02:58:22Z","started_at":"2026-09-25T02:58:22Z","completed_at":"2026-09-25T05:11:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T05:22:00.694850231Z","timing_metadata_checked_at":"2026-09-25T05:22:00.694850231Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T05:22:00.697561124Z","region_metadata_checked_at":"2026-09-25T05:22:00.697561124Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0af5dbf6a9def2fe85e7bcf72eae31b6a3966151fb64519a608882930168e19c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318164392644608","occurred_at":"2026-09-25T02:58:22Z","failed_at":"e2e","signature_id":"fe76c6b3b8bcd7b776b70ad4fdbbcc743eda7378548d4371f57952d35a7644da","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pqhwsk\" for cluster \"cluster-candidate-4-20-pqhwsk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001061ba0\u003e: \n failed to create NodePool np-pqhwsk: failed waiting for nodepool=\"np-pqhwsk\" for cluster \"cluster-candidate-4-20-pqhwsk\" in resourcegroup=\"rg-candidate-4-20-4hfcxgvg226p\" to finish creating: GET https://rp.j2644608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ca27daa-447f-4425-a737-272daf2be568\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ca27daa-447f-4425-a737-272daf2be568\",\n \"name\": \"2ca27daa-447f-4425-a737-272daf2be568\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T04:05:18.601636409Z\",\n \"endTime\": \"2026-09-25T04:24:23.185191796Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-4hfcxgvg226p-candidate-pqhwsk--managed/providers/Microsoft.Compute/virtualMachines/j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-4hfcxgvg226p-candidate-pqhwsk--managed/providers/Microsoft.Compute/virtualMachines/j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pqhwsk\" for cluster \"cluster-candidate-4-20-pqhwsk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001061ba0\u003e: \n failed to create NodePool np-pqhwsk: failed waiting for nodepool=\"np-pqhwsk\" for cluster \"cluster-candidate-4-20-pqhwsk\" in resourcegroup=\"rg-candidate-4-20-4hfcxgvg226p\" to finish creating: GET https://rp.j2644608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ca27daa-447f-4425-a737-272daf2be568\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ca27daa-447f-4425-a737-272daf2be568\",\n \"name\": \"2ca27daa-447f-4425-a737-272daf2be568\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T04:05:18.601636409Z\",\n \"endTime\": \"2026-09-25T04:24:23.185191796Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-4hfcxgvg226p-candidate-pqhwsk--managed/providers/Microsoft.Compute/virtualMachines/j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-4hfcxgvg226p-candidate-pqhwsk--managed/providers/Microsoft.Compute/virtualMachines/j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eacd13fa6af2b06f56ce1295197593af50aee51808239f944fd6ca6273414a50","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"be2e2663063f16f9968b4764ddaafe65d02b751fdbd39b15c176fe98c5ed2253","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318164392644608","occurred_at":"2026-09-25T02:58:22Z","failed_at":"alert","signature_id":"4935221bcf4cd8ebd4a711d2d33e112ac42f8c8ae29a231910ec6bb2fc856310","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T04:28:16Z\n Ended: 2026-09-25T04:34:17Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2644608-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2644608-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","eacd13fa6af2b06f56ce1295197593af50aee51808239f944fd6ca6273414a50"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7155/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103314299182125056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7155,"pr_state":"closed","pr_sha":"2c285c27d181371596fd7063d957601efb266472","final_merged_sha":"b99e022318555865ffcb4306017cec183e68d902","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T02:43:01Z","started_at":"2026-09-25T02:43:01Z","completed_at":"2026-09-25T03:30:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T03:37:49.223949209Z","timing_metadata_checked_at":"2026-09-25T03:37:49.223949209Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T03:37:49.226713995Z","region_metadata_checked_at":"2026-09-25T03:37:49.226713995Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a5cb80af0f20bde4b00e31cdeb9a0dba5e8bbcff2601eb3c32a6a22feb06592a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7155/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103314299182125056","occurred_at":"2026-09-25T02:43:01Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T02:56:01.375Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T02:56:03.005Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T02:56:03.628Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d65a0948f2d84197bbbf6951bd15e5094221d06b6fa0387b4ca487587287aa93 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2125056%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd65a0948f2d84197bbbf6951bd15e5094221d06b6fa0387b4ca487587287aa93\ntime=2026-09-25T02:58:34.184Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2125056/providers/Microsoft.Resources/deployments/d65a0948f2d84197bbbf6951bd15e5094221d06b6fa0387b4ca487587287aa93/operationStatuses/08584113019222692213\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2125056/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2125056/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j2125056\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 4dbe9c6e-f586-4cab-af30-db5b9ff45239:9/25/2026 2:57:28 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T03:00:34.191Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T03:00:34.888Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T03:00:35.650Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=839dfc3ba303c251cf4e6cf34912141fb13991c2481665648345ec648e3b7007 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2125056%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F839dfc3ba303c251cf4e6cf34912141fb13991c2481665648345ec648e3b7007\ntime=2026-09-25T03:17:39.729Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2125056/providers/Microsoft.Resources/deployments/839dfc3ba303c251cf4e6cf34912141fb13991c2481665648345ec648e3b7007/operationStatuses/08584113016502442636\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2125056/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2125056/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j2125056\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report ef3307f7-7b9d-4276-b102-5ec4d200c7c4:9/25/2026 3:17:03 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T03:19:39.731Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T03:19:40.553Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T03:19:41.903Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=a8f83de670c39868606eacb2ceb05ab140c1d718907c7454f53294f7465599ab portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2125056%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fa8f83de670c39868606eacb2ceb05ab140c1d718907c7454f53294f7465599ab\ntime=2026-09-25T03:26:01.098Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7153/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103310372827041792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7153,"pr_state":"closed","pr_sha":"db4db03d9ece2571a9f23d96b0038ebfb951da6a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T02:27:25Z","started_at":"2026-09-25T02:27:25Z","completed_at":"2026-09-25T02:45:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T02:55:41.243598213Z","timing_metadata_checked_at":"2026-09-25T02:55:41.243598213Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T02:55:41.246910802Z","region_metadata_checked_at":"2026-09-25T02:55:41.246910802Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4123855fd36ac9dde35bed8c159da62743f93655ac5767368b4b2a69739acd3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7153/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103310372827041792","occurred_at":"2026-09-25T02:27:25Z","failed_at":"provision","signature_id":"eb3a097ae9ea09f1b87574d34137bdc9344882caed893e38ceda6a45fc8e1166","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/maestroMqttDataConnection","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T02:39:19.860Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n Kind: ARM\\n Template: templates/maestro-mqtt-data-connection.bicep\\n Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-25T02:39:20.257Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection\ntime=2026-09-25T02:39:20.970Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection deployment=67437eb973c49795934fc79733f3a01017eeb2994048cb050f5f7bf549795e78 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-kusto-us%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F67437eb973c49795934fc79733f3a01017eeb2994048cb050f5f7bf549795e78\ntime=2026-09-25T02:39:51.103Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/67437eb973c49795934fc79733f3a01017eeb2994048cb050f5f7bf549795e78/operationStatuses/08584113029248069745\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"EventHubValidationErrorFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"[BadRequest] Validation Errors found: Table does not exist\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7772a8fa7766108b00c98006e84756f50f97d8f9c7242d532de03696ce3d923c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code EventHubValidationErrorFound; detail message [BadRequest] Validation Errors found: Table does not exist","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7772a8fa7766108b00c98006e84756f50f97d8f9c7242d532de03696ce3d923c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7154/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103307352265461760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7154,"pr_state":"closed","pr_sha":"be2bb8717d94fe3eeb4284138768e6aec0ce5b1c","final_merged_sha":"be2bb8717d94fe3eeb4284138768e6aec0ce5b1c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T02:15:24Z","started_at":"2026-09-25T02:15:24Z","completed_at":"2026-09-25T04:27:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T04:40:49.499973601Z","timing_metadata_checked_at":"2026-09-25T04:40:49.499973601Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T04:40:49.504286487Z","region_metadata_checked_at":"2026-09-25T04:40:49.504286487Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7148,"pr_state":"closed","pr_sha":"c3e1a8ef598ecb7f85e0ae9a62013997e2cf959c","final_merged_sha":"c3e1a8ef598ecb7f85e0ae9a62013997e2cf959c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T02:07:11Z","started_at":"2026-09-25T02:07:11Z","completed_at":"2026-09-25T04:19:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T04:29:56.145068304Z","timing_metadata_checked_at":"2026-09-25T04:29:56.145068304Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T04:29:56.147528296Z","region_metadata_checked_at":"2026-09-25T04:29:56.147528296Z"},"failed_at":["alert","e2e"],"failed_test_count":29,"occurrences":[{"row_id":"03da520a635dc06950adc363d3e461bf7815a4ba4708cf10b5cc9a11426166d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"alert","signature_id":"73c465500cc6b96cb01f7f0c6b77eae53537748367fdb4379569cc20c60be2b5","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-25T03:35:30Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7908608-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7908608/providers/microsoft.monitor/accounts/services-j7908608\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-dq66f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-25T04:01:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7908608-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7908608/providers/microsoft.monitor/accounts/services-j7908608\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-dq66f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"0717c479a3fe4b80b475b2fc3e6c0e0eb7b943cdc15656340353343bfb34e1d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"ce5516485360ff5329c2949a660bc2dc211057187d80e612e660b2bc332176ec","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0e720\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7br46fldbmb6\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f3f8911-b947-4e57-8acd-cc2888bba8ff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f3f8911-b947-4e57-8acd-cc2888bba8ff\",\n \"name\": \"0f3f8911-b947-4e57-8acd-cc2888bba8ff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:42.645827565Z\",\n \"endTime\": \"2026-09-25T03:28:43.634764956Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0e720\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7br46fldbmb6\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f3f8911-b947-4e57-8acd-cc2888bba8ff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f3f8911-b947-4e57-8acd-cc2888bba8ff\",\n \"name\": \"0f3f8911-b947-4e57-8acd-cc2888bba8ff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:42.645827565Z\",\n \"endTime\": \"2026-09-25T03:28:43.634764956Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0ce6fc6f3174fa32aa1a95a611f4310eab2b4f86dd36e45c9807f93701ee6c0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"8dfd2d9aee99bd2f3c4b3476e89dfe81012a96f67d220278846c2a4450bb6f6d","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000edaa20\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-p68zdj9gxmmx\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a134b2e-976a-413d-bce8-4af5d68d73d8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a134b2e-976a-413d-bce8-4af5d68d73d8\",\n \"name\": \"8a134b2e-976a-413d-bce8-4af5d68d73d8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:51.42313966Z\",\n \"endTime\": \"2026-09-25T03:27:53.61766454Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000edaa20\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-p68zdj9gxmmx\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a134b2e-976a-413d-bce8-4af5d68d73d8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a134b2e-976a-413d-bce8-4af5d68d73d8\",\n \"name\": \"8a134b2e-976a-413d-bce8-4af5d68d73d8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:51.42313966Z\",\n \"endTime\": \"2026-09-25T03:27:53.61766454Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"20524a5eeb7c9aa333f64bec326deccc2a4cd1905c0d3ae8b56fc0f3a6fcdc83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"d0c345a150f7ac5287d9623336261f3cc51366673893d4967734a655c83ca35b","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b64ba0\u003e: \n failed to create HCP cluster np-downgrade-7jfrcx: failed waiting for cluster=\"np-downgrade-7jfrcx\" in resourcegroup=\"rg-np-downgrade-7jfrcx-4hmfb6mlsx7s\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2aede227-e60f-45ef-9764-867bae3ede9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2aede227-e60f-45ef-9764-867bae3ede9e\",\n \"name\": \"2aede227-e60f-45ef-9764-867bae3ede9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T03:20:19.72569999Z\",\n \"endTime\": \"2026-09-25T03:49:24.24417195Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b64ba0\u003e: \n failed to create HCP cluster np-downgrade-7jfrcx: failed waiting for cluster=\"np-downgrade-7jfrcx\" in resourcegroup=\"rg-np-downgrade-7jfrcx-4hmfb6mlsx7s\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2aede227-e60f-45ef-9764-867bae3ede9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2aede227-e60f-45ef-9764-867bae3ede9e\",\n \"name\": \"2aede227-e60f-45ef-9764-867bae3ede9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T03:20:19.72569999Z\",\n \"endTime\": \"2026-09-25T03:49:24.24417195Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2064307b6ef4225cb8bc234be2220f767d4afad5d8a954e0b69ac4ca430d2546","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"a7ae7c35d502ce2a586d4148a43a08195b73b6c1310e43fa75f4c2e07e6936c0","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001acad40\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-zvjvpbz4jg4z\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1dd914d2-a004-4fe1-8701-bdb2cb75bb80\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1dd914d2-a004-4fe1-8701-bdb2cb75bb80\",\n \"name\": \"1dd914d2-a004-4fe1-8701-bdb2cb75bb80\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:45.137839085Z\",\n \"endTime\": \"2026-09-25T03:27:53.499253257Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001acad40\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-zvjvpbz4jg4z\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1dd914d2-a004-4fe1-8701-bdb2cb75bb80\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1dd914d2-a004-4fe1-8701-bdb2cb75bb80\",\n \"name\": \"1dd914d2-a004-4fe1-8701-bdb2cb75bb80\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:45.137839085Z\",\n \"endTime\": \"2026-09-25T03:27:53.499253257Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"229c2f4cc36f83c651e4a33419264c8e806b0f49925ca672a604e6d62896743e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"b4611c05f2efe3971dd158997c067a5564d4eaa50f357276c0c5460a0c6e6a6c","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8f100\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-7h6qt949b7r2\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4553a11d-460d-4c19-9cf1-b46eb91f1684\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4553a11d-460d-4c19-9cf1-b46eb91f1684\",\n \"name\": \"4553a11d-460d-4c19-9cf1-b46eb91f1684\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:40.646090429Z\",\n \"endTime\": \"2026-09-25T03:28:43.487307022Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8f100\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-7h6qt949b7r2\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4553a11d-460d-4c19-9cf1-b46eb91f1684\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4553a11d-460d-4c19-9cf1-b46eb91f1684\",\n \"name\": \"4553a11d-460d-4c19-9cf1-b46eb91f1684\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:40.646090429Z\",\n \"endTime\": \"2026-09-25T03:28:43.487307022Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2cdacb1df51c7edf4961dfcaf395f9ecc03f6f913727f5f132d153e42256c801","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"4d22a3664875424009f741d660d7e41b3b5f053f4621ecdadc098445ae1ea436","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c22a0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-vg28mwtv88bj\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3eb923d-a560-4839-8932-23550d08a65e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3eb923d-a560-4839-8932-23550d08a65e\",\n \"name\": \"c3eb923d-a560-4839-8932-23550d08a65e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:43.83209061Z\",\n \"endTime\": \"2026-09-25T03:27:43.747183624Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c22a0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-vg28mwtv88bj\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3eb923d-a560-4839-8932-23550d08a65e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3eb923d-a560-4839-8932-23550d08a65e\",\n \"name\": \"c3eb923d-a560-4839-8932-23550d08a65e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:43.83209061Z\",\n \"endTime\": \"2026-09-25T03:27:43.747183624Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"45a83087af3524e21a07379a0264d145937cd3e244731256b07d945b07becbbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"df135d2d019527778b2ef6a5cf1bdb06fcbe11f52a39c8f44008c1c975f75b8a","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002484680\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-hx7bzrspmbbr\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f597b880-abc9-415f-b64a-db05503ef743\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f597b880-abc9-415f-b64a-db05503ef743\",\n \"name\": \"f597b880-abc9-415f-b64a-db05503ef743\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:52.851458252Z\",\n \"endTime\": \"2026-09-25T03:27:53.765532616Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002484680\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-hx7bzrspmbbr\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f597b880-abc9-415f-b64a-db05503ef743\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f597b880-abc9-415f-b64a-db05503ef743\",\n \"name\": \"f597b880-abc9-415f-b64a-db05503ef743\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:52.851458252Z\",\n \"endTime\": \"2026-09-25T03:27:53.765532616Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4b8fe0b44a4590da0fcecec60bdfbe19b42c8d3d5ea6fc16b9dbd6becb016aec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"85907840a45a76d3a4f061bee5f17411e53f0903b97983b05c738378f6bfed7b","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-4x78ml\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00151e440\u003e: \n failed to create HCP cluster np-noedge-4x78ml: failed waiting for cluster=\"np-noedge-4x78ml\" in resourcegroup=\"rg-np-noedge-4x78ml-qbsdchjbclsx\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92ed9d26-0b0a-4a27-8260-4c85f8114d28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92ed9d26-0b0a-4a27-8260-4c85f8114d28\",\n \"name\": \"92ed9d26-0b0a-4a27-8260-4c85f8114d28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:53.257571888Z\",\n \"endTime\": \"2026-09-25T03:28:53.531492573Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-4x78ml\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00151e440\u003e: \n failed to create HCP cluster np-noedge-4x78ml: failed waiting for cluster=\"np-noedge-4x78ml\" in resourcegroup=\"rg-np-noedge-4x78ml-qbsdchjbclsx\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92ed9d26-0b0a-4a27-8260-4c85f8114d28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92ed9d26-0b0a-4a27-8260-4c85f8114d28\",\n \"name\": \"92ed9d26-0b0a-4a27-8260-4c85f8114d28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:53.257571888Z\",\n \"endTime\": \"2026-09-25T03:28:53.531492573Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4e5b9ecc051282659b9d918d88b09374a6d19aa7983d454180f22eb2463a524f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"5fde953f368ed57b002630d1dcf62b22a35d8dc85adbbb537362ccc55a78382e","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c7d0c0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wd4jdc9qs4qd\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a1c2265-97d6-4805-bd6e-425c02d68d98\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a1c2265-97d6-4805-bd6e-425c02d68d98\",\n \"name\": \"3a1c2265-97d6-4805-bd6e-425c02d68d98\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:52.159712074Z\",\n \"endTime\": \"2026-09-25T03:27:53.626252681Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c7d0c0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wd4jdc9qs4qd\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a1c2265-97d6-4805-bd6e-425c02d68d98\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a1c2265-97d6-4805-bd6e-425c02d68d98\",\n \"name\": \"3a1c2265-97d6-4805-bd6e-425c02d68d98\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:52.159712074Z\",\n \"endTime\": \"2026-09-25T03:27:53.626252681Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6023f2be432b68bbcf28e90ebfb800932fd4cc9607e873cf59e6d85de9261e77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"4e3b027db47198851dfc1a758d2d2d219a8eaacd78a6c47f12f5ec4d6ce3fe3e","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b93400\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-52z7lm5cpsfc\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf9bc76e-0a3a-482d-ac8a-4b837680d30a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf9bc76e-0a3a-482d-ac8a-4b837680d30a\",\n \"name\": \"cf9bc76e-0a3a-482d-ac8a-4b837680d30a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T03:02:01.866786194Z\",\n \"endTime\": \"2026-09-25T03:31:04.34796657Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b93400\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-52z7lm5cpsfc\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf9bc76e-0a3a-482d-ac8a-4b837680d30a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf9bc76e-0a3a-482d-ac8a-4b837680d30a\",\n \"name\": \"cf9bc76e-0a3a-482d-ac8a-4b837680d30a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T03:02:01.866786194Z\",\n \"endTime\": \"2026-09-25T03:31:04.34796657Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"646d85d7cf36bf65edf4b4b60e00d3b6056344b2d1b246ce432c939756b6404d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"10468b2864f877d33d6b2b49ecda8314a65d0f555b80b14355c0af87148ea4dc","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-lbsddf with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8c360\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-lbsddf: failed waiting for cluster=\"np-version-upgrade-cluster-lbsddf\" in resourcegroup=\"rg-np-version-upgrade-lbsddf-tvrng8dc2496\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfd3731e-5e9b-458d-928f-6eea5509a636\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfd3731e-5e9b-458d-928f-6eea5509a636\",\n \"name\": \"dfd3731e-5e9b-458d-928f-6eea5509a636\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:11.878359495Z\",\n \"endTime\": \"2026-09-25T03:28:13.63075435Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-lbsddf with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8c360\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-lbsddf: failed waiting for cluster=\"np-version-upgrade-cluster-lbsddf\" in resourcegroup=\"rg-np-version-upgrade-lbsddf-tvrng8dc2496\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfd3731e-5e9b-458d-928f-6eea5509a636\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfd3731e-5e9b-458d-928f-6eea5509a636\",\n \"name\": \"dfd3731e-5e9b-458d-928f-6eea5509a636\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:11.878359495Z\",\n \"endTime\": \"2026-09-25T03:28:13.63075435Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6d0ac62a2575d68ba85570624eeae242fe18b77c09229b502c8cccdfa05b1a03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"65801013559be7bb9d6cbeb6a5a16a7e16d2f3de96a3532698d9587676ab9892","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e1cd60\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vkgbzhdrxpl5\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/69672e8b-b923-4dd3-8b08-1d57558b7a59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/69672e8b-b923-4dd3-8b08-1d57558b7a59\",\n \"name\": \"69672e8b-b923-4dd3-8b08-1d57558b7a59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:02.668900293Z\",\n \"endTime\": \"2026-09-25T03:28:03.786147694Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e1cd60\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vkgbzhdrxpl5\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/69672e8b-b923-4dd3-8b08-1d57558b7a59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/69672e8b-b923-4dd3-8b08-1d57558b7a59\",\n \"name\": \"69672e8b-b923-4dd3-8b08-1d57558b7a59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:02.668900293Z\",\n \"endTime\": \"2026-09-25T03:28:03.786147694Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6f3354a13a9f4d9aa99d27fff1fc28642d13318565dc740b20e3b2f4ddb402d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"2adebc884c23131da1cb9fad9e2d7884273de1cfd5d252f83e841782e4127ee6","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd86c0\u003e: \n failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-xcts8564xmdp\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2365e304-fdb7-46fb-adcb-c90df765ef44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2365e304-fdb7-46fb-adcb-c90df765ef44\",\n \"name\": \"2365e304-fdb7-46fb-adcb-c90df765ef44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:53.401589102Z\",\n \"endTime\": \"2026-09-25T03:28:53.66288459Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd86c0\u003e: \n failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-xcts8564xmdp\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2365e304-fdb7-46fb-adcb-c90df765ef44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2365e304-fdb7-46fb-adcb-c90df765ef44\",\n \"name\": \"2365e304-fdb7-46fb-adcb-c90df765ef44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:53.401589102Z\",\n \"endTime\": \"2026-09-25T03:28:53.66288459Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"835a14948f282291a27c1cf462ece770a9228f021b43718299ff42e0c39750bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"e6be7f265e4d9ec98cda597c7fc56a6dd0f69b9755ac6a89ac9f3a58964311c5","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-dvjdts\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001714a20\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-dvjdts: failed waiting for cluster=\"cp-ystream-upgrade-4-21-dvjdts\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-kjpcwx85fvbv\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e6d19ca-041a-4033-bd49-81e35567eb49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e6d19ca-041a-4033-bd49-81e35567eb49\",\n \"name\": \"5e6d19ca-041a-4033-bd49-81e35567eb49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:53.264164454Z\",\n \"endTime\": \"2026-09-25T03:27:53.39266999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-dvjdts\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001714a20\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-dvjdts: failed waiting for cluster=\"cp-ystream-upgrade-4-21-dvjdts\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-kjpcwx85fvbv\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e6d19ca-041a-4033-bd49-81e35567eb49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e6d19ca-041a-4033-bd49-81e35567eb49\",\n \"name\": \"5e6d19ca-041a-4033-bd49-81e35567eb49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:53.264164454Z\",\n \"endTime\": \"2026-09-25T03:27:53.39266999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d79eb84631fd5a9c0b1c31d21a63bb61a6e92d9ea2a98505596a46c83851083","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"alert","signature_id":"e1f51160fdcdfe2935875cac6be61e62b5d5cc074bef6b194f6075bfe9248be6","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-25T03:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7908608-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7908608/providers/microsoft.monitor/accounts/services-j7908608\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-dq66f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-25T03:26:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7908608-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7908608/providers/microsoft.monitor/accounts/services-j7908608\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-dq66f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"9643cbfd40a0e9f56d174161fb892b9253aa3c2f5d4378d89d1b2e86dd8362bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"alert","signature_id":"23ecffc8f4b7c15d8ff10d4272fefe60a14a999143507f8a644a0af9081b9865","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-25T03:00:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7908608-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-78648d9c8d-tkfsj\", region=\"westus3\", severity=\"warning\"\n Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-78648d9c8d-tkfsj has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"9cab7106f5decc26bc52c6b4ec219774766d16217c7526186d8d2b7d248caafa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"22cf1e89265cf76ccbe0f2e98d44db46e9295497f14a6e8d772e76120a16dece","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cccea0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-w9hkzhnrs46x\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e53f084-0836-43b7-813a-0f6f679c2c37\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e53f084-0836-43b7-813a-0f6f679c2c37\",\n \"name\": \"7e53f084-0836-43b7-813a-0f6f679c2c37\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:01.927806669Z\",\n \"endTime\": \"2026-09-25T03:28:03.710517796Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cccea0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-w9hkzhnrs46x\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e53f084-0836-43b7-813a-0f6f679c2c37\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e53f084-0836-43b7-813a-0f6f679c2c37\",\n \"name\": \"7e53f084-0836-43b7-813a-0f6f679c2c37\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:01.927806669Z\",\n \"endTime\": \"2026-09-25T03:28:03.710517796Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a1172ee476c7c9d70e69fa301d07c9ac9658ddd0bd192ab2697e9527a311e4b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"b604b39072c658d2ce9100ad7c38c8840c663636b574cfacbc58b604945878f6","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0002b24e0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-c98m9xz88ffj\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fabaa68d-2b69-4912-9b65-fec658af9c09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fabaa68d-2b69-4912-9b65-fec658af9c09\",\n \"name\": \"fabaa68d-2b69-4912-9b65-fec658af9c09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:12.137301983Z\",\n \"endTime\": \"2026-09-25T03:28:13.45329639Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0002b24e0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-c98m9xz88ffj\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fabaa68d-2b69-4912-9b65-fec658af9c09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fabaa68d-2b69-4912-9b65-fec658af9c09\",\n \"name\": \"fabaa68d-2b69-4912-9b65-fec658af9c09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:12.137301983Z\",\n \"endTime\": \"2026-09-25T03:28:13.45329639Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a98f70c84f53614f8dfa61c2b3f8557801569af1c593e02446327b3e20f876b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"3c5cb05e3d81db1356f6fde7ea0b62ca1261edaad85a2845e3986d212eb6d0d0","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001378120\u003e: \n failed to create HCP cluster np-dg-minor-pjrpz9: failed waiting for cluster=\"np-dg-minor-pjrpz9\" in resourcegroup=\"rg-np-dg-minor-pjrpz9-vmct4xsrlfcg\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/198b5d46-5118-4945-870d-f5c79c47e025\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/198b5d46-5118-4945-870d-f5c79c47e025\",\n \"name\": \"198b5d46-5118-4945-870d-f5c79c47e025\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T03:21:40.065204195Z\",\n \"endTime\": \"2026-09-25T03:50:44.261232799Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001378120\u003e: \n failed to create HCP cluster np-dg-minor-pjrpz9: failed waiting for cluster=\"np-dg-minor-pjrpz9\" in resourcegroup=\"rg-np-dg-minor-pjrpz9-vmct4xsrlfcg\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/198b5d46-5118-4945-870d-f5c79c47e025\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/198b5d46-5118-4945-870d-f5c79c47e025\",\n \"name\": \"198b5d46-5118-4945-870d-f5c79c47e025\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T03:21:40.065204195Z\",\n \"endTime\": \"2026-09-25T03:50:44.261232799Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ae0bc3abb0ba3c4af0330708cd3757d7dacdc10bbdd8ca4ee75555eed7738936","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"5b5cc3f029412de7c3201b6a756aa4d38e26a2737b8850dcb0ac5cda518e740f","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b9a600\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ck5ph7pk6zhr\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c34336f9-c477-4264-ac6a-85c56ca91228\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c34336f9-c477-4264-ac6a-85c56ca91228\",\n \"name\": \"c34336f9-c477-4264-ac6a-85c56ca91228\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T03:01:18.68523424Z\",\n \"endTime\": \"2026-09-25T03:30:24.247909224Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b9a600\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ck5ph7pk6zhr\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c34336f9-c477-4264-ac6a-85c56ca91228\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c34336f9-c477-4264-ac6a-85c56ca91228\",\n \"name\": \"c34336f9-c477-4264-ac6a-85c56ca91228\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T03:01:18.68523424Z\",\n \"endTime\": \"2026-09-25T03:30:24.247909224Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aea59be31d209b21bd10c45c695f9866d3d1e12789a5b706a364ecc3083e28bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"alert","signature_id":"0a3b060ab5af73558d8c0a74ad7ce0917f462e5e9705c25366e434181ee85d37","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-25T04:00:50Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j7908608-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7908608/providers/microsoft.monitor/accounts/services-j7908608\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of completed cluster create (install) operations on ci01-j7908608-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors6h30m fired"}},{"row_id":"b1620e050fe59206677dd69cb9ce2d86c4bb54d6fc3bfb48137f98f832a0069f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"2b35303720f768483a8d89608537e77ec72ff8a2dbd2f717eb6913e0b2640354","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010509a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-lwvn6v6j7ch5\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4f91ea60-c164-467c-adf6-0a2094ec136f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4f91ea60-c164-467c-adf6-0a2094ec136f\",\n \"name\": \"4f91ea60-c164-467c-adf6-0a2094ec136f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:55.185356163Z\",\n \"endTime\": \"2026-09-25T03:29:03.527933292Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010509a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-lwvn6v6j7ch5\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4f91ea60-c164-467c-adf6-0a2094ec136f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4f91ea60-c164-467c-adf6-0a2094ec136f\",\n \"name\": \"4f91ea60-c164-467c-adf6-0a2094ec136f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:55.185356163Z\",\n \"endTime\": \"2026-09-25T03:29:03.527933292Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c60c5d9f3874c16429454ba8c666f2348909717a2be22b20091cc15522d6e5d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"772419b8b1da0a62ac9ae138152e4f3ffa0ce55f12a177fc428c05293a97832e","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c4c200\u003e: \n failed to create HCP cluster ea-list4fkmwn: failed waiting for cluster=\"ea-list4fkmwn\" in resourcegroup=\"ea-list-vcr8pjqdp4tn\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7d0df8a-17fa-4e9e-9582-b7699684b235\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7d0df8a-17fa-4e9e-9582-b7699684b235\",\n \"name\": \"f7d0df8a-17fa-4e9e-9582-b7699684b235\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:44.015346047Z\",\n \"endTime\": \"2026-09-25T03:27:53.775985477Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c4c200\u003e: \n failed to create HCP cluster ea-list4fkmwn: failed waiting for cluster=\"ea-list4fkmwn\" in resourcegroup=\"ea-list-vcr8pjqdp4tn\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7d0df8a-17fa-4e9e-9582-b7699684b235\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7d0df8a-17fa-4e9e-9582-b7699684b235\",\n \"name\": \"f7d0df8a-17fa-4e9e-9582-b7699684b235\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:44.015346047Z\",\n \"endTime\": \"2026-09-25T03:27:53.775985477Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d6b5e13c223ba60fc1edaf1538b42cbf1663e5e12004795f3597bc3141fe1e63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"249f1acbefa9c6f20ccd9712fe5a00c23629ae49fcc09ce711b79b24927d99a7","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006df780\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-l2fmpdwgxxr7\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2db7371-9a1e-4204-9ef5-6f3d7af49462\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2db7371-9a1e-4204-9ef5-6f3d7af49462\",\n \"name\": \"c2db7371-9a1e-4204-9ef5-6f3d7af49462\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:44.421041016Z\",\n \"endTime\": \"2026-09-25T03:27:53.514379412Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006df780\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-l2fmpdwgxxr7\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2db7371-9a1e-4204-9ef5-6f3d7af49462\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2db7371-9a1e-4204-9ef5-6f3d7af49462\",\n \"name\": \"c2db7371-9a1e-4204-9ef5-6f3d7af49462\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:44.421041016Z\",\n \"endTime\": \"2026-09-25T03:27:53.514379412Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"da76fa6d9bb441aa93f7c1ad61acc6f4df181498a58c61bdb384e3c1705384e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"718929c4f69b8ebaee796119ffa00a30eaba1b5c81dc9549343d0c9d2b8dc5f0","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017a2450\u003e: \n GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/df23530c-dab3-4ba4-822b-f0b2b2b4aa56\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/df23530c-dab3-4ba4-822b-f0b2b2b4aa56\",\n \"name\": \"df23530c-dab3-4ba4-822b-f0b2b2b4aa56\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:33.427733744Z\",\n \"endTime\": \"2026-09-25T03:28:33.687116408Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017a2450\u003e: \n GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/df23530c-dab3-4ba4-822b-f0b2b2b4aa56\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/df23530c-dab3-4ba4-822b-f0b2b2b4aa56\",\n \"name\": \"df23530c-dab3-4ba4-822b-f0b2b2b4aa56\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:59:33.427733744Z\",\n \"endTime\": \"2026-09-25T03:28:33.687116408Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e4f2097754a42748945c3887228c57566826c54482a18d1844459ef23fd36dce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"40caa6e5062642065971953209cdd8c9b48b360fb5a1ddc9bc485ca1336da7db","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cc62a0\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-qkjzkx24wjrd\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48976594-9647-401f-8c41-c29741f2bec5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48976594-9647-401f-8c41-c29741f2bec5\",\n \"name\": \"48976594-9647-401f-8c41-c29741f2bec5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:45.306263975Z\",\n \"endTime\": \"2026-09-25T03:27:53.691358323Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cc62a0\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-qkjzkx24wjrd\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48976594-9647-401f-8c41-c29741f2bec5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48976594-9647-401f-8c41-c29741f2bec5\",\n \"name\": \"48976594-9647-401f-8c41-c29741f2bec5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:58:45.306263975Z\",\n \"endTime\": \"2026-09-25T03:27:53.691358323Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f7c86038f6cbf5274752c2473e9055bc377b02b7e2d1fb9fefc66277e2a720f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"e2e","signature_id":"c8027d91e14911182feeba1f8b9551aac196e52d333c551356e8711f64c54816","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013ec620\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gv8mrhx9vqnl\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cacc23c3-2de1-4c02-982e-d6ad30996aaa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cacc23c3-2de1-4c02-982e-d6ad30996aaa\",\n \"name\": \"cacc23c3-2de1-4c02-982e-d6ad30996aaa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T03:25:12.399098514Z\",\n \"endTime\": \"2026-09-25T03:54:14.278226893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013ec620\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gv8mrhx9vqnl\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cacc23c3-2de1-4c02-982e-d6ad30996aaa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cacc23c3-2de1-4c02-982e-d6ad30996aaa\",\n \"name\": \"cacc23c3-2de1-4c02-982e-d6ad30996aaa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T03:25:12.399098514Z\",\n \"endTime\": \"2026-09-25T03:54:14.278226893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fec499ee1fbe99b71c3a97ab505ec1477c393ab2fe08f47771d04e275b19ffed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","failed_at":"alert","signature_id":"40d5ad535cc1dad15220e6b19910e3eb9b33324cfc93904c2a5a3b66cda18505","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T03:34:52Z\n Ended: 2026-09-25T03:43:52Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7908608-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-78567cf5bf-qmbvj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-78567cf5bf-qmbvj (namespace kube-applier) on cluster ci01-j7908608-mgmt-1 has averaged 138.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":29,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0"],"matched_occurrences":29,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305260238901248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"f7eae70a7727145890f4c333ef421260214cc407","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T02:07:06Z","started_at":"2026-09-25T02:07:06Z","completed_at":"2026-09-25T03:07:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T03:16:54.282328019Z","timing_metadata_checked_at":"2026-09-25T03:16:54.282328019Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T03:16:54.286444606Z","region_metadata_checked_at":"2026-09-25T03:16:54.286444606Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0db01f1e9ad08a0ab815c8c6668a8e223a8791e9ad8064a8eed71210b0c8caea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305260238901248","occurred_at":"2026-09-25T02:07:06Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T02:32:18.185Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T02:32:19.885Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T02:32:20.742Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=294f72eb6aa57edb055ced5b6bc70daa4caa80e004fee48d425a69b38a7752e8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8901248%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F294f72eb6aa57edb055ced5b6bc70daa4caa80e004fee48d425a69b38a7752e8\ntime=2026-09-25T02:34:51.729Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8901248/providers/Microsoft.Resources/deployments/294f72eb6aa57edb055ced5b6bc70daa4caa80e004fee48d425a69b38a7752e8/operationStatuses/08584113033453081554\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8901248/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8901248/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8901248\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report dda65ea3-cb7b-4960-934f-32656d920b79:9/25/2026 2:33:44 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T02:36:51.730Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T02:36:52.610Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T02:36:53.280Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f3da3c9d9692b5e4f7c34fd879ac26b2534c01decac19cd72c34c2d6da757aac portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8901248%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff3da3c9d9692b5e4f7c34fd879ac26b2534c01decac19cd72c34c2d6da757aac\ntime=2026-09-25T02:52:58.252Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8901248/providers/Microsoft.Resources/deployments/f3da3c9d9692b5e4f7c34fd879ac26b2534c01decac19cd72c34c2d6da757aac/operationStatuses/08584113030725767633\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8901248/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8901248/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j8901248\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 54e87199-165f-40ba-980d-8603e86b7e27:9/25/2026 2:52:34 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T02:54:58.259Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T02:54:59.191Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T02:54:59.929Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5bc124f0a4ec5afe1c14762a8ced0f7648eb8b0e7a1a72703b83fb58c2f7abbe portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8901248%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5bc124f0a4ec5afe1c14762a8ced0f7648eb8b0e7a1a72703b83fb58c2f7abbe\ntime=2026-09-25T03:02:17.897Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305015505457152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"a092ebd92da1578187635cb299d5ea86ead6f3c8","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T02:06:07Z","started_at":"2026-09-25T02:06:07Z","completed_at":"2026-09-25T04:27:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T04:40:49.213894828Z","timing_metadata_checked_at":"2026-09-25T04:40:49.213894828Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T04:40:49.217551916Z","region_metadata_checked_at":"2026-09-25T04:40:49.217551916Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0cab4f1e4c9da96ec61790579c5289dab1e0d8fb7724faa92eb9bf67553b306c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305015505457152","occurred_at":"2026-09-25T02:06:07Z","failed_at":"alert","signature_id":"52816a71ef2a03b49578fdbb7a916c3c123b4226a4221c05983979cb58643e9c","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T03:24:55Z\n Ended: 2026-09-25T03:44:57Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5457152-svc\", component=\"backend\", name=\"datadump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"486ad78d6a54a1f37e1f59aa11f23dc7bc310f76b181410bd1bdc46ac93acb14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305015505457152","occurred_at":"2026-09-25T02:06:07Z","failed_at":"e2e","signature_id":"23feb3a96cb75059920f0d80a8340e60dd48c1eb6d7d1330a687410267280342","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001c6a5a0\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4dxpnt9zrzkg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4dxpnt9zrzkg, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001c6a5a0\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4dxpnt9zrzkg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4dxpnt9zrzkg, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-w8sl5z8nrnnt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-w8sl5z8nrnnt, error: context deadline exceeded"}},{"row_id":"aabc14396ce2cedb84c31e60f52e20ca1b91aadd03840496d9dc333ccc32db09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305015505457152","occurred_at":"2026-09-25T02:06:07Z","failed_at":"e2e","signature_id":"4e82558e3f130c0b3cf620c520d17c6d0e6517b368b13c167cb42967e19e28ba","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:80]: failed to create cluster customer resources for cordon test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b98ae0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cordon-hcp-cluster-rck8ht\" in resourcegroup=\"rg-cordon-test-tlds99sfrm5j\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cordon-test-tlds99sfrm5j/providers/Microsoft.Resources/deployments/customer-infra-cordon-hcp-cluster-rck8ht/operationStatuses/08584113017077419903\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:80]: failed to create cluster customer resources for cordon test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b98ae0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cordon-hcp-cluster-rck8ht\" in resourcegroup=\"rg-cordon-test-tlds99sfrm5j\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cordon-test-tlds99sfrm5j/providers/Microsoft.Resources/deployments/customer-infra-cordon-hcp-cluster-rck8ht/operationStatuses/08584113017077419903\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103304771547959296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7118,"pr_state":"closed","pr_sha":"179ae75a7e3eff02b934146a76b63b8d8d485193","final_merged_sha":"f23942d30dc47ec7edeebb24ec30f183a59ea2fd","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T02:05:09Z","started_at":"2026-09-25T02:05:09Z","completed_at":"2026-09-25T04:23:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T04:29:55.559939982Z","timing_metadata_checked_at":"2026-09-25T04:29:55.559939982Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T04:29:55.563660671Z","region_metadata_checked_at":"2026-09-25T04:29:55.563660671Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"5ee882ed05ebe0029a9fe0c1f7a56b95d7efd4d295d7b0ca346bd871a750138a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103304771547959296","occurred_at":"2026-09-25T02:05:09Z","failed_at":"alert","signature_id":"4f8032ceda13192ec98e764421a26949f70feecc56ae688c668ca100e00819d7","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T03:31:21Z\n Ended: 2026-09-25T03:37:22Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7959296-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7959296-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"91e10fc8bd04e66ef3618c6f036488f249be6f7a9e62b3fc1ee3691b0399bb7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103304771547959296","occurred_at":"2026-09-25T02:05:09Z","failed_at":"alert","signature_id":"bfb35c392292260a733e3e716557db76a69fa86926d85051968134ab2390355d","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T03:26:19Z\n Ended: 2026-09-25T03:40:21Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7959296-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-7dc989b6d9-rg6jc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-7dc989b6d9-rg6jc (namespace kube-applier) on cluster ci01-j7959296-mgmt-1 has averaged 138.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7149/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103281892986982400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7149,"pr_state":"closed","pr_sha":"75f4b3d73fbc760e8b69df0db4067b8702fb2556","final_merged_sha":"75f4b3d73fbc760e8b69df0db4067b8702fb2556","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T00:34:11Z","started_at":"2026-09-25T00:34:11Z","completed_at":"2026-09-25T01:34:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T01:42:39.445662356Z","timing_metadata_checked_at":"2026-09-25T01:42:39.445662356Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T01:42:39.449143545Z","region_metadata_checked_at":"2026-09-25T01:42:39.449143545Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d389e47d3697b2baf6da9574929558abbef317d35b8af7f502587b2936cc5717","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7149/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103281892986982400","occurred_at":"2026-09-25T00:34:11Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T00:58:15.246Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T00:58:17.058Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T00:58:17.734Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=a90317428421ae4a9831143605c3348b6779fcfd8539c4fadcd16d587055f70f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6982400%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fa90317428421ae4a9831143605c3348b6779fcfd8539c4fadcd16d587055f70f\ntime=2026-09-25T01:00:48.341Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6982400/providers/Microsoft.Resources/deployments/a90317428421ae4a9831143605c3348b6779fcfd8539c4fadcd16d587055f70f/operationStatuses/08584113089882240440\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6982400/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6982400/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6982400\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report ccaa9de3-26fc-4b4f-839e-13c85fb2a293:9/25/2026 12:59:19 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T01:02:48.344Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T01:02:49.003Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T01:02:49.979Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=fdf83f6ca29febe849f187987401e0b085ce4a2e7288b9b7a941094a7fe31467 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6982400%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ffdf83f6ca29febe849f187987401e0b085ce4a2e7288b9b7a941094a7fe31467\ntime=2026-09-25T01:19:23.882Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6982400/providers/Microsoft.Resources/deployments/fdf83f6ca29febe849f187987401e0b085ce4a2e7288b9b7a941094a7fe31467/operationStatuses/08584113087161022541\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6982400/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6982400/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6982400\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 1a3f07a7-85e1-48b5-a7d4-08fd8e4f1190:9/25/2026 1:18:48 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T01:21:23.883Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T01:21:24.448Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T01:21:25.238Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=ab17f73dfb0d00f69653f8c0cfd8877518a95f1d5844f4569e3f5b6b2e476362 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6982400%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fab17f73dfb0d00f69653f8c0cfd8877518a95f1d5844f4569e3f5b6b2e476362\ntime=2026-09-25T01:28:14.966Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103277014529085440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"f7eae70a7727145890f4c333ef421260214cc407","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-25T00:14:51Z","started_at":"2026-09-25T00:14:51Z","completed_at":"2026-09-25T01:22:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T01:31:54.600809667Z","timing_metadata_checked_at":"2026-09-25T01:31:54.600809667Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T01:31:54.612622026Z","region_metadata_checked_at":"2026-09-25T01:31:54.612622026Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a9e565dc50993f8469f0bbb21fd35966a5f8db7c5a482b818fee94e6208383a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103277014529085440","occurred_at":"2026-09-25T00:14:51Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-25T00:44:13.666Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T00:44:15.063Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T00:44:15.907Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=aae5d1e8ad1eca1703d0336abd5a03be66fcdd7a6d397c537a2c60dd89bbe7fa portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9085440%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Faae5d1e8ad1eca1703d0336abd5a03be66fcdd7a6d397c537a2c60dd89bbe7fa\ntime=2026-09-25T00:46:46.486Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9085440/providers/Microsoft.Resources/deployments/aae5d1e8ad1eca1703d0336abd5a03be66fcdd7a6d397c537a2c60dd89bbe7fa/operationStatuses/08584113098301384143\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9085440/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9085440/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9085440\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 25e2b9e6-b085-4a73-a10a-b806d3b393e9:9/25/2026 12:45:31 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T00:48:46.487Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T00:48:52.290Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T00:48:53.060Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=b2ec9a2ac155b0f6c84ad43227576eb6c420ed1204f9a196bb2a3593c7ca7353 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9085440%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb2ec9a2ac155b0f6c84ad43227576eb6c420ed1204f9a196bb2a3593c7ca7353\ntime=2026-09-25T01:04:59.273Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9085440/providers/Microsoft.Resources/deployments/b2ec9a2ac155b0f6c84ad43227576eb6c420ed1204f9a196bb2a3593c7ca7353/operationStatuses/08584113095528788265\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9085440/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9085440/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9085440\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 86281f9a-3569-4410-92d7-50a5ac169a78:9/25/2026 1:04:34 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T01:06:59.275Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T01:07:00.063Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T01:07:01.160Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=b88fd4fe6f6a5534beb570442dcdc3c5e851c5fec4cac7be02753929753dc5ed portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9085440%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb88fd4fe6f6a5534beb570442dcdc3c5e851c5fec4cac7be02753929753dc5ed\ntime=2026-09-25T01:14:13.386Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"int","summary":{"total_runs":7,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103598326791278592","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T21:31:38Z","started_at":"2026-09-25T21:31:38Z","completed_at":"2026-09-25T23:52:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T00:12:25.481026831Z","timing_metadata_checked_at":"2026-09-26T00:12:25.481026831Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T00:12:25.485814316Z","region_metadata_checked_at":"2026-09-26T00:12:25.485814316Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103550178567917568","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T18:20:19Z","started_at":"2026-09-25T18:20:19Z","completed_at":"2026-09-25T20:39:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T20:52:34.415166804Z","timing_metadata_checked_at":"2026-09-25T20:52:34.415166804Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T20:52:34.417962396Z","region_metadata_checked_at":"2026-09-25T20:52:34.417962396Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103500347635929088","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T15:01:22Z","started_at":"2026-09-25T15:01:22Z","completed_at":"2026-09-25T17:18:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T17:34:03.455730885Z","timing_metadata_checked_at":"2026-09-25T17:34:03.455730885Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T17:34:03.459086976Z","region_metadata_checked_at":"2026-09-25T17:34:03.459086976Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"40aa2ba53f2aef741b0ce0aaf893b24065b7b4b63550e8552950e011b0263ed3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103500347635929088","occurred_at":"2026-09-25T15:01:22Z","failed_at":"e2e","signature_id":"2374477c920f24f39cbd859778b4b5423d9aa24bc3495ca463c798a21c8c8e87","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000599de0\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-kcj4l5\" in resourcegroup=\"rg-np-skip-minor-kcj4l5-45s8nhm4k78b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da564fd3-aea3-4a41-b171-3969372d023d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da564fd3-aea3-4a41-b171-3969372d023d\",\n \"name\": \"da564fd3-aea3-4a41-b171-3969372d023d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T16:39:04.653915252Z\",\n \"endTime\": \"2026-09-25T16:58:09.716908828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kcj4l5-45s8nhm4k78b-np-sm-kcj4l5--managed/providers/Microsoft.Compute/virtualMachines/k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kcj4l5-45s8nhm4k78b-np-sm-kcj4l5--managed/providers/Microsoft.Compute/virtualMachines/k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000599de0\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-kcj4l5\" in resourcegroup=\"rg-np-skip-minor-kcj4l5-45s8nhm4k78b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da564fd3-aea3-4a41-b171-3969372d023d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da564fd3-aea3-4a41-b171-3969372d023d\",\n \"name\": \"da564fd3-aea3-4a41-b171-3969372d023d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T16:39:04.653915252Z\",\n \"endTime\": \"2026-09-25T16:58:09.716908828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kcj4l5-45s8nhm4k78b-np-sm-kcj4l5--managed/providers/Microsoft.Compute/virtualMachines/k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kcj4l5-45s8nhm4k78b-np-sm-kcj4l5--managed/providers/Microsoft.Compute/virtualMachines/k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bfd08cdd46bd2b7366c25bc24686fba64cd529977b75f42a98cbe09926d75184","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103500347635929088","occurred_at":"2026-09-25T15:01:22Z","failed_at":"e2e","signature_id":"e1f4ee9af599a77b687540e9f4697061e341edc13f3416df00ca7e8b696944ff","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f18e80\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-p9kv2wz66ng2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d3308e-222c-40d8-a1be-8c057c26bab1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d3308e-222c-40d8-a1be-8c057c26bab1\",\n \"name\": \"82d3308e-222c-40d8-a1be-8c057c26bab1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:45:43.486812846Z\",\n \"endTime\": \"2026-09-25T16:14:49.560161271Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f18e80\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-p9kv2wz66ng2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d3308e-222c-40d8-a1be-8c057c26bab1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d3308e-222c-40d8-a1be-8c057c26bab1\",\n \"name\": \"82d3308e-222c-40d8-a1be-8c057c26bab1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T15:45:43.486812846Z\",\n \"endTime\": \"2026-09-25T16:14:49.560161271Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"53d6a9f6efeca45a9da0e1f8eb60e93e70de646c11d05978730da3382fb8e964","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254","53d6a9f6efeca45a9da0e1f8eb60e93e70de646c11d05978730da3382fb8e964"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103448928220352512","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-25T11:37:59Z","started_at":"2026-09-25T11:37:59Z","completed_at":"2026-09-25T13:59:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T14:12:39.754358513Z","timing_metadata_checked_at":"2026-09-25T14:12:39.754358513Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T14:12:39.757426004Z","region_metadata_checked_at":"2026-09-25T14:12:39.757426004Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103397522356572160","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T08:13:43Z","started_at":"2026-09-25T08:13:43Z","completed_at":"2026-09-25T10:18:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T10:30:30.754722619Z","timing_metadata_checked_at":"2026-09-25T10:30:30.754722619Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T10:30:30.758208008Z","region_metadata_checked_at":"2026-09-25T10:30:30.758208008Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9c53ba9a71bdfa205b67e4c612156308ce86dd9ed10bf70b233e8635f01db0e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103397522356572160","occurred_at":"2026-09-25T08:13:43Z","failed_at":"e2e","signature_id":"4b38b160a22968120417e984b979b00dd8bb78826d4b38714c0eb589ea58bd2f","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00075fc80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-dnhw57q2zhsr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\",\n \"name\": \"9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T08:30:53.138668141Z\",\n \"endTime\": \"2026-09-25T08:49:53.966422643Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-dnhw57q2zhsr--managed/providers/Microsoft.Compute/virtualMachines/q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-dnhw57q2zhsr--managed/providers/Microsoft.Compute/virtualMachines/q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00075fc80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-dnhw57q2zhsr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\",\n \"name\": \"9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T08:30:53.138668141Z\",\n \"endTime\": \"2026-09-25T08:49:53.966422643Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-dnhw57q2zhsr--managed/providers/Microsoft.Compute/virtualMachines/q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-dnhw57q2zhsr--managed/providers/Microsoft.Compute/virtualMachines/q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cd3b62302145a4f193634c04292cd7c3b0522163606e6fe1f23b2a097a00d90e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103397522356572160","occurred_at":"2026-09-25T08:13:43Z","failed_at":"e2e","signature_id":"b009af289e732a1b6a4d96b5624455888514f82524439fc4e668740147e9e47a","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c02c0\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2mw49n77wnb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e57de6e4-350f-44ca-8989-0fa815b3fd2f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e57de6e4-350f-44ca-8989-0fa815b3fd2f\",\n \"name\": \"e57de6e4-350f-44ca-8989-0fa815b3fd2f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T08:27:10.94071903Z\",\n \"endTime\": \"2026-09-25T08:46:13.959695236Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-2mw49n77wnb2--managed/providers/Microsoft.Compute/virtualMachines/y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-2mw49n77wnb2--managed/providers/Microsoft.Compute/virtualMachines/y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c02c0\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2mw49n77wnb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e57de6e4-350f-44ca-8989-0fa815b3fd2f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e57de6e4-350f-44ca-8989-0fa815b3fd2f\",\n \"name\": \"e57de6e4-350f-44ca-8989-0fa815b3fd2f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T08:27:10.94071903Z\",\n \"endTime\": \"2026-09-25T08:46:13.959695236Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-2mw49n77wnb2--managed/providers/Microsoft.Compute/virtualMachines/y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-2mw49n77wnb2--managed/providers/Microsoft.Compute/virtualMachines/y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103361291237199872","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T05:49:44Z","started_at":"2026-09-25T05:49:44Z","completed_at":"2026-09-25T08:08:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T08:18:47.51872031Z","timing_metadata_checked_at":"2026-09-25T08:18:47.51872031Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T08:18:47.521194301Z","region_metadata_checked_at":"2026-09-25T08:18:47.521194301Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7e2edfcbfb6ac03ecec6c3f12d0711e76729f99e0c71eacee7353ad38a8bcd93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103361291237199872","occurred_at":"2026-09-25T05:49:44Z","failed_at":"e2e","signature_id":"26173be16e73b2272c2dda84a4b010bd8ef49864c818286cf90989c383c22b93","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pt5qmx\" for cluster \"cluster-candidate-5-0-pt5qmx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c10900\u003e: \n failed to create NodePool np-pt5qmx: failed starting nodepool creation \"np-pt5qmx\" for cluster \"cluster-candidate-5-0-pt5qmx\" in resourcegroup=\"rg-candidate-5-0-9clwz8rf6599\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-9clwz8rf6599/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-pt5qmx/nodePools/np-pt5qmx\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot update resource while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-9clwz8rf6599/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-pt5qmx/nodePools/np-pt5qmx\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pt5qmx\" for cluster \"cluster-candidate-5-0-pt5qmx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c10900\u003e: \n failed to create NodePool np-pt5qmx: failed starting nodepool creation \"np-pt5qmx\" for cluster \"cluster-candidate-5-0-pt5qmx\" in resourcegroup=\"rg-candidate-5-0-9clwz8rf6599\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-9clwz8rf6599/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-pt5qmx/nodePools/np-pt5qmx\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot update resource while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-9clwz8rf6599/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-pt5qmx/nodePools/np-pt5qmx\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"46b0598b5ebe8f06dc5f7cfd83233a53219d8dfdc2ad61e72855646ca0260a66","failure_pattern":"ERROR CODE: Conflict; detail message Cannot update resource while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["46b0598b5ebe8f06dc5f7cfd83233a53219d8dfdc2ad61e72855646ca0260a66"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103303905172525056","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T02:01:43Z","started_at":"2026-09-25T02:01:43Z","completed_at":"2026-09-25T04:24:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T04:29:56.340156911Z","timing_metadata_checked_at":"2026-09-25T04:29:56.340156911Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T04:29:56.343166302Z","region_metadata_checked_at":"2026-09-25T04:29:56.343166302Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1d81a42ca2463d1175410f06c66301700ab11f4ca2717cf06cc6629ebc8e4d1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103303905172525056","occurred_at":"2026-09-25T02:01:43Z","failed_at":"e2e","signature_id":"f35b530f26f0d3fedd585cc4516c1a10ecb44588c174c5fe5a05614b69732b79","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007453e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-58b5tglh77bz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28fe561e-2b93-4934-b274-0e50cb5390d6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28fe561e-2b93-4934-b274-0e50cb5390d6\",\n \"name\": \"28fe561e-2b93-4934-b274-0e50cb5390d6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:36:40.600765804Z\",\n \"endTime\": \"2026-09-25T02:55:41.450889323Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-58b5tglh77bz--managed/providers/Microsoft.Compute/virtualMachines/x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-58b5tglh77bz--managed/providers/Microsoft.Compute/virtualMachines/x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007453e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-58b5tglh77bz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28fe561e-2b93-4934-b274-0e50cb5390d6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28fe561e-2b93-4934-b274-0e50cb5390d6\",\n \"name\": \"28fe561e-2b93-4934-b274-0e50cb5390d6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:36:40.600765804Z\",\n \"endTime\": \"2026-09-25T02:55:41.450889323Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-58b5tglh77bz--managed/providers/Microsoft.Compute/virtualMachines/x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-58b5tglh77bz--managed/providers/Microsoft.Compute/virtualMachines/x1c7o2k3x9l0n1u-np-1-kxng9-xrnnb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T23:00:16Z","started_at":"2026-09-25T23:00:16Z","completed_at":"2026-09-26T01:10:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T01:16:24.953174015Z","timing_metadata_checked_at":"2026-09-26T01:16:24.953174015Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T01:16:24.955873507Z","region_metadata_checked_at":"2026-09-26T01:16:24.955873507Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"11e2755fded4b4fb244e1b05b56b07bdde54354fb9823971e47f3ad4f743b479","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"d0fabc45709d827c15fe3c2a6fbbced1e9f6e7903fe8a15397d348a35770aa86","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-sf8c2g\" for cluster \"cluster-nightly-4-20-sf8c2g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000949500\u003e: \n failed to create NodePool np-sf8c2g: failed starting nodepool creation \"np-sf8c2g\" for cluster \"cluster-nightly-4-20-sf8c2g\" in resourcegroup=\"rg-nightly-4-20-7swt4w2kzhfs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-7swt4w2kzhfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-sf8c2g/nodePools/np-sf8c2g\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-sf8c2g\" for cluster \"cluster-nightly-4-20-sf8c2g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000949500\u003e: \n failed to create NodePool np-sf8c2g: failed starting nodepool creation \"np-sf8c2g\" for cluster \"cluster-nightly-4-20-sf8c2g\" in resourcegroup=\"rg-nightly-4-20-7swt4w2kzhfs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-7swt4w2kzhfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-sf8c2g/nodePools/np-sf8c2g\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"12ca2db9276c1a0e078592d3a6f541d167d3ca42d0c8740dd570dbd23ae79d31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"ab0087d314f58a4462c0a5c4e09f0320ebe8f8af617d9a58fb7a9d27fefcf6e1","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxlj58f6 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-5tckvstjpzw8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0011fb4a0\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxlj58f6 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-5tckvstjpzw8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0011fb4a0\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}},{"row_id":"260771658d329fa7e3eb666cdb34de32a5f2f90b0cc0f91c01c46a3547a40091","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"fe9849f3545963512f1ece2f25d17c32ca1e7e435e9e95ea90adfb3ea562ce8a","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000700c40\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-xqscjv\" in resourcegroup=\"rg-np-version-upgrade-xqscjv-cbdfvf855m7r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-xqscjv-cbdfvf855m7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-xqscjv/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000700c40\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-xqscjv\" in resourcegroup=\"rg-np-version-upgrade-xqscjv-cbdfvf855m7r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-xqscjv-cbdfvf855m7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-xqscjv/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2b51e56e9f0c89b903c054bb2b4249ccb9fc55f82fbc15e6367b15cabbc01f3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"9d32c182a0e9b653b758b4108d7d2bed1ed38f4bb428003742a75f25756d9bd9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-7q8vqcfshfst/cluster-nightly-5-1-htcp7l should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000acaa60\u003e: \n failed to create HCP cluster cluster-nightly-5-1-htcp7l: failed waiting for cluster=\"cluster-nightly-5-1-htcp7l\" in resourcegroup=\"rg-nightly-5-1-7q8vqcfshfst\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3dbd0e68-03bb-49be-ad53-7a10f8750a36\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3dbd0e68-03bb-49be-ad53-7a10f8750a36\",\n \"name\": \"3dbd0e68-03bb-49be-ad53-7a10f8750a36\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T23:32:03.179098884Z\",\n \"endTime\": \"2026-09-26T00:01:10.002965076Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-7q8vqcfshfst/cluster-nightly-5-1-htcp7l should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000acaa60\u003e: \n failed to create HCP cluster cluster-nightly-5-1-htcp7l: failed waiting for cluster=\"cluster-nightly-5-1-htcp7l\" in resourcegroup=\"rg-nightly-5-1-7q8vqcfshfst\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3dbd0e68-03bb-49be-ad53-7a10f8750a36\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3dbd0e68-03bb-49be-ad53-7a10f8750a36\",\n \"name\": \"3dbd0e68-03bb-49be-ad53-7a10f8750a36\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T23:32:03.179098884Z\",\n \"endTime\": \"2026-09-26T00:01:10.002965076Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4528b434af694f8025e1b1474e4ef626ed5489dcd675669ca845a20091661f31","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"44eb20b79590b5396b4ff369b7ecfc44ecf70bf9e6b48bf6a6eed7a0336f4519","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"325840a2b42ebb8d501c0b8e4f87b5bd1159c7995917009439b796ba33def4de","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00078b060\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gb52bkffcwft\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/db1ab9c2-9577-48f8-9f15-b41d89774bb7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/db1ab9c2-9577-48f8-9f15-b41d89774bb7\",\n \"name\": \"db1ab9c2-9577-48f8-9f15-b41d89774bb7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T23:59:44.019265481Z\",\n \"endTime\": \"2026-09-26T00:18:50.149741392Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00078b060\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gb52bkffcwft\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/db1ab9c2-9577-48f8-9f15-b41d89774bb7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/db1ab9c2-9577-48f8-9f15-b41d89774bb7\",\n \"name\": \"db1ab9c2-9577-48f8-9f15-b41d89774bb7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T23:59:44.019265481Z\",\n \"endTime\": \"2026-09-26T00:18:50.149741392Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gb52bkffcwft\" to finish creating: GET https://management"}},{"row_id":"525e67c8150744bfc10d40397f54e97ff0d445270bbcd41e1b757989f1ad301e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"1323fc4bbb4f8bec0749ff5a18b557ada5537b1e2d9ac0d4f832c4644c0b23b7","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013e4db0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-t66lgcklqd6q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013e4db0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-t66lgcklqd6q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5697c56643aa6271f5f7b3df9252402ceac346700299dde69b5790722d177a8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"2d0664bc68395570897d3edf86ebea891697dd9f67b9c208c184ac4acd358176","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0027024b0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-zd52hqnd5jgk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0027024b0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-zd52hqnd5jgk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7a82c0d9f413bc3395d6753f757ce7ddd94fafc17162d76d1cb75a1cb4cba060","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"25c9c8629d36608562eacc178345af00071b7c15f03f2b7932324336ce809df8","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001025cc0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-zlflh9\" in resourcegroup=\"rg-np-skip-minor-zlflh9-nz2rc2k8lntd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-zlflh9-nz2rc2k8lntd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-zlflh9/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001025cc0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-zlflh9\" in resourcegroup=\"rg-np-skip-minor-zlflh9-nz2rc2k8lntd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-zlflh9-nz2rc2k8lntd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-zlflh9/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a1572c99497a5463166bfdc4d3a0f4d4cabcde422d5c469624684804de77cdf9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"0da3e1c733b3a346e6237f6eb60d4c8b4b0a1f2ab3382bc7c6e0a96d1c2b7c4c","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011d8600\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-8l7wjgzcr5kh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011d8600\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-8l7wjgzcr5kh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a4eb0de0fd3139f0b64e7baf72aa41024b6254dbf64dbfa1da38e0d6b40c9b8f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"06fc4248c731a78ec09d70b92471beff4048830ea286a825e3ad2027e0ced359","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00117a060\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-8xvvbqc2kcgw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00117a060\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-8xvvbqc2kcgw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a6b9f3d4ee14061039e74cf39c114948f7f97b85ffca3de43a353c29eadb0ce1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"5b65b5d543e8d31941a31a2864c015c8c52f0f301461e1c272635b2b14df1af9","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001419e20\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-lp7f8j\" resourcegroup=\"rg-np-dg-minor-lp7f8j-kb9jlc2jfsr2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-lp7f8j-kb9jlc2jfsr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-lp7f8j/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001419e20\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-lp7f8j\" resourcegroup=\"rg-np-dg-minor-lp7f8j-kb9jlc2jfsr2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-lp7f8j-kb9jlc2jfsr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-lp7f8j/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b1541c74c2b05f8b4f4482721b1d40ed40b9df89ea2cf7eb43cdccda5e61f55b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"76257e848298b1062f3dec33bd08c56cb39fdc88fccdde36fe3867b8cef0eeff","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002b89d70\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-skvrt7nhhnxp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002b89d70\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-skvrt7nhhnxp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b51846a6a9029fdd90a2e5d80e29549d6130f3225e922334a3ed7dbbd69cc0e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"c3ce37d99ec460bf1b759e73ea22337f6eacbb332a70e22befe3dd9f6c0bf0d2","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000bc5170\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000bc5170\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","4528b434af694f8025e1b1474e4ef626ed5489dcd675669ca845a20091661f31","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103303533200674816","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T02:00:14Z","started_at":"2026-09-25T02:00:14Z","completed_at":"2026-09-25T04:23:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T04:29:56.563330832Z","timing_metadata_checked_at":"2026-09-25T04:29:56.563330832Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T04:29:56.566085024Z","region_metadata_checked_at":"2026-09-25T04:29:56.566085024Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"16eb2536c73927ba2b818e6ee18a5189d766409ed8545fb10965f94483a64708","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103303533200674816","occurred_at":"2026-09-25T02:00:14Z","failed_at":"e2e","signature_id":"e07f78fb07fe2b29f2a237f72f104117e81b0766f04a34ec1cc5af886ce79354","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008ef760\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-jbgszp\" in resourcegroup=\"rg-np-version-upgrade-jbgszp-szxzcp5vvtll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/767328b3-e719-4dce-a9f8-d13c4b4802eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/767328b3-e719-4dce-a9f8-d13c4b4802eb\",\n \"name\": \"767328b3-e719-4dce-a9f8-d13c4b4802eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T03:20:28.674763184Z\",\n \"endTime\": \"2026-09-25T03:39:37.254782397Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008ef760\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-jbgszp\" in resourcegroup=\"rg-np-version-upgrade-jbgszp-szxzcp5vvtll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/767328b3-e719-4dce-a9f8-d13c4b4802eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/767328b3-e719-4dce-a9f8-d13c4b4802eb\",\n \"name\": \"767328b3-e719-4dce-a9f8-d13c4b4802eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T03:20:28.674763184Z\",\n \"endTime\": \"2026-09-25T03:39:37.254782397Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2d954fdcc2dd34c28c85dd2a34ff5c07948328abed9880e0ac9e44e7e1af65cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103303533200674816","occurred_at":"2026-09-25T02:00:14Z","failed_at":"e2e","signature_id":"78dffc2ed4c7c4ccb6dfde345d2214bde1baff9b62e1668576e19c12f9366096","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc000ff01b0\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-d8dncw5q6xr2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cccec67-451f-41c1-957a-1453bb0ac735\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cccec67-451f-41c1-957a-1453bb0ac735\",\n \"name\": \"4cccec67-451f-41c1-957a-1453bb0ac735\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:15:20.351974211Z\",\n \"endTime\": \"2026-09-25T02:34:26.038545442Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n encountered errors downloading console logs:\n - VM \"cilium-cluster-cilium-np-44b92-qd24g\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-44b92-qd24g\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d8dncw5q6xr2--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-44b92-qd24g/retrieveBootDiagnosticsData\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: OperationNotAllowed\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"OperationNotAllowed\",\n \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-44b92-qd24g' since another operation is in progress.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc000ff01b0\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-d8dncw5q6xr2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cccec67-451f-41c1-957a-1453bb0ac735\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cccec67-451f-41c1-957a-1453bb0ac735\",\n \"name\": \"4cccec67-451f-41c1-957a-1453bb0ac735\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:15:20.351974211Z\",\n \"endTime\": \"2026-09-25T02:34:26.038545442Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n encountered errors downloading console logs:\n - VM \"cilium-cluster-cilium-np-44b92-qd24g\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-44b92-qd24g\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d8dncw5q6xr2--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-44b92-qd24g/retrieveBootDiagnosticsData\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: OperationNotAllowed\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"OperationNotAllowed\",\n \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-44b92-qd24g' since another operation is in progress.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7646eda388fcf6391d44cd32bf5781942e0ad533f17bf8d059ce1a303c08693","failure_pattern":"ERROR CODE: InternalServerError; detail code OperationNotAllowed; detail message Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-44b92-\u003cid\u003e' since another operation is in progress.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5fb91d977db2e1eea05df949956e1407ce73d0e43cc09f1540df739253d8ddfb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103303533200674816","occurred_at":"2026-09-25T02:00:14Z","failed_at":"e2e","signature_id":"f5bee8b440fafcbcf3226fb257766f44fc56668b176e888a34752eb1baf4b876","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00013d940\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pv4shddpb7qx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/979880cc-8a28-44f5-8525-bb6028b5328e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/979880cc-8a28-44f5-8525-bb6028b5328e\",\n \"name\": \"979880cc-8a28-44f5-8525-bb6028b5328e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:12:08.565557217Z\",\n \"endTime\": \"2026-09-25T02:31:16.095644453Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00013d940\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pv4shddpb7qx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/979880cc-8a28-44f5-8525-bb6028b5328e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/979880cc-8a28-44f5-8525-bb6028b5328e\",\n \"name\": \"979880cc-8a28-44f5-8525-bb6028b5328e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T02:12:08.565557217Z\",\n \"endTime\": \"2026-09-25T02:31:16.095644453Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ff22d8898829b0aded3a5e42d6a40f2e4dca2fd5d023779837186fafe8b360f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103303533200674816","occurred_at":"2026-09-25T02:00:14Z","failed_at":"e2e","signature_id":"3d1829567b78ae4d2efc76eba5be75f4a96678a24f6fcf274e485617885c4edd","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxsjrhvt and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-722np9bhtr46/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0007b7cd0\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxsjrhvt and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-722np9bhtr46/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0007b7cd0\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","a7646eda388fcf6391d44cd32bf5781942e0ad533f17bf8d059ce1a303c08693"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"stg","summary":{"total_runs":5,"failed_runs":5,"runs_with_occurrences":5,"runs_with_failure_pattern_matches":5,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T23:00:16Z","started_at":"2026-09-25T23:00:16Z","completed_at":"2026-09-26T01:39:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T01:47:57.786753998Z","timing_metadata_checked_at":"2026-09-26T01:47:57.786753998Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T01:47:57.789415589Z","region_metadata_checked_at":"2026-09-26T01:47:57.789415589Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"3cfc6e469033e8c7ff7aef3d3190629ec71704a0c365babee7cdcba3468637d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"4e21bc12c99ecec673a96eb7593881309ac7de9775f283ab767dd467523d44d1","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010fe8a0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-srk662g7xx9b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010fe8a0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-srk662g7xx9b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"429198ab4a3fe4f4042bc6666e2924d10280d8c21f2cf26e156795bca01ca3a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"8e4966742e7d4d0ad960487d3edf5beaf995ef862658f56dd2fa158c7259a509","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001659e40\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-ftxd2m\" resourcegroup=\"rg-np-dg-minor-ftxd2m-fl7b6d52hcz6\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-ftxd2m-fl7b6d52hcz6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-ftxd2m/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001659e40\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-ftxd2m\" resourcegroup=\"rg-np-dg-minor-ftxd2m-fl7b6d52hcz6\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-ftxd2m-fl7b6d52hcz6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-ftxd2m/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"609892f8229bb167b4a9008f66b92d31f9db11acfae3c6568c3c64203f90213f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"0e7801135f075953518765c1bbbddc68cb14d38ab7ce1effafb5ee7011cd9767","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00088f9e0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-7kjs5ps99kbv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00088f9e0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-7kjs5ps99kbv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8185460bf8cfbaed2e3eb3ec0623b3f492df7c3378e69d49a15f6e4db9d8c4b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"50e75bf1403da3027cc0838aae74e5b39ec954431257679b542bd194fafd38de","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0020da1e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-z8zt66k27vt9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0020da1e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-z8zt66k27vt9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a359e3182d08900e8277ba5b68c7508004021618f9f12c046952377520c8e0ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"e5fa85fb92e02ccfc9e63d3a7fab3f53bfabfbab7d6cbda05b9d8eae9f85d1a4","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d68200\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-2xv5gn\" in resourcegroup=\"rg-np-skip-minor-2xv5gn-mr64n6ncrgps\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-2xv5gn-mr64n6ncrgps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-2xv5gn/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d68200\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-2xv5gn\" in resourcegroup=\"rg-np-skip-minor-2xv5gn-mr64n6ncrgps\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-2xv5gn-mr64n6ncrgps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-2xv5gn/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a9d22f6efddc63fa76268aefb739389756feb72647dc0bbee1d27fa01d292f61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"4c4499c70bf7e7fbd7efdcba08b18c93c68baac4fc5629cdcc576459752fe59d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-tklhx4zzdx6h/cluster-nightly-5-1-vjz6jm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e262a0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-vjz6jm: failed waiting for cluster=\"cluster-nightly-5-1-vjz6jm\" in resourcegroup=\"rg-nightly-5-1-tklhx4zzdx6h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9063e9-f978-4377-b5d9-c9222622088e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9063e9-f978-4377-b5d9-c9222622088e\",\n \"name\": \"5f9063e9-f978-4377-b5d9-c9222622088e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T00:24:56.11201607Z\",\n \"endTime\": \"2026-09-26T00:54:00.860881838Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-tklhx4zzdx6h/cluster-nightly-5-1-vjz6jm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e262a0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-vjz6jm: failed waiting for cluster=\"cluster-nightly-5-1-vjz6jm\" in resourcegroup=\"rg-nightly-5-1-tklhx4zzdx6h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9063e9-f978-4377-b5d9-c9222622088e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9063e9-f978-4377-b5d9-c9222622088e\",\n \"name\": \"5f9063e9-f978-4377-b5d9-c9222622088e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T00:24:56.11201607Z\",\n \"endTime\": \"2026-09-26T00:54:00.860881838Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c071429a287be0f254f4f0e70d2ef61574322b99e904b0a2496ef6afa776b8ab","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e0f530d7ca4bf88bbcdf36b1fa14cc7b1491e4c98e073d9e7b2268cff053b59c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"0cb35b890f581d959b6853f4aea9845c15baa5df736da136cda474135f2e5079","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dt4hdm\" for cluster \"cluster-nightly-4-20-dt4hdm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000600e20\u003e: \n failed to create NodePool np-dt4hdm: failed starting nodepool creation \"np-dt4hdm\" for cluster \"cluster-nightly-4-20-dt4hdm\" in resourcegroup=\"rg-nightly-4-20-mbwzhwdx5487\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-mbwzhwdx5487/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-dt4hdm/nodePools/np-dt4hdm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dt4hdm\" for cluster \"cluster-nightly-4-20-dt4hdm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000600e20\u003e: \n failed to create NodePool np-dt4hdm: failed starting nodepool creation \"np-dt4hdm\" for cluster \"cluster-nightly-4-20-dt4hdm\" in resourcegroup=\"rg-nightly-4-20-mbwzhwdx5487\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-mbwzhwdx5487/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-dt4hdm/nodePools/np-dt4hdm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"e2711cbb0560cc9dc1819a5ef79045cddadc3ad63d01a4f71263ca7c08133b13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"7025c972aa4849457e5ddbe4ef9be30261f8964b5b0a0be1395928ce0cdc81ea","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a169e0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-8wcm2h\" in resourcegroup=\"rg-np-version-upgrade-8wcm2h-qhfg24hmjm4w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-8wcm2h-qhfg24hmjm4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-8wcm2h/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a169e0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-8wcm2h\" in resourcegroup=\"rg-np-version-upgrade-8wcm2h-qhfg24hmjm4w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-8wcm2h-qhfg24hmjm4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-8wcm2h/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"eecae9f31e48f8809d95d6dbfb4e60068e486383f31e5fefb67d87b238a9db16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"a7c8a4561bc6869b3844c32d80edf46aa9a5ca5311095d7f67b3d5dce4de9b9d","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001409500\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001409500\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"f3af8be8b921d4c71b224066f20e4d9f30d130278705e602e75bec952d507b3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"db14bfab8e3a48be52d9cfa18b18664997f945501b054b544b3db7caf90c40c3","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00163a600\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-t26vzx6s4brs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00163a600\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-t26vzx6s4brs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ff854ac651708c9dde6bf058ba6843295d5d8ebab0a9ba751f670a32a5d2ab53","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","failed_at":"e2e","signature_id":"a7b728efe3485e59acc90fa1890af7ee6cdc0e407befeaca1f81b294e8fd1008","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0016277d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-7c4t9f4jq285/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0016277d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-7c4t9f4jq285/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","c071429a287be0f254f4f0e70d2ef61574322b99e904b0a2496ef6afa776b8ab"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103603313239920640","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T21:51:27Z","started_at":"2026-09-25T21:51:27Z","completed_at":"2026-09-25T23:59:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T00:12:25.666754752Z","timing_metadata_checked_at":"2026-09-26T00:12:25.666754752Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T00:12:25.671562537Z","region_metadata_checked_at":"2026-09-26T00:12:25.671562537Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2185467053b3cb2f255be4d43567fd0cb0e2106fd9767eaf079c0708a9506653","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103603313239920640","occurred_at":"2026-09-25T21:51:27Z","failed_at":"e2e","signature_id":"ea79c06297d62c705cc8405c02627178ecb3397453f3b4a80d1e574e28dc5c93","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0015ba9f0\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-2rtwrg4pjxl2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/88abacf9-a5e0-4248-b32c-83d6cd062714\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/88abacf9-a5e0-4248-b32c-83d6cd062714\",\n \"name\": \"88abacf9-a5e0-4248-b32c-83d6cd062714\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T22:23:02.156910064Z\",\n \"endTime\": \"2026-09-25T22:42:06.143243532Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-2klmb-6glqn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2rtwrg4pjxl2--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2klmb-6glqn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-2klmb-6glqn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2rtwrg4pjxl2--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2klmb-6glqn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0015ba9f0\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-2rtwrg4pjxl2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/88abacf9-a5e0-4248-b32c-83d6cd062714\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/88abacf9-a5e0-4248-b32c-83d6cd062714\",\n \"name\": \"88abacf9-a5e0-4248-b32c-83d6cd062714\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T22:23:02.156910064Z\",\n \"endTime\": \"2026-09-25T22:42:06.143243532Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-2klmb-6glqn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2rtwrg4pjxl2--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2klmb-6glqn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-2klmb-6glqn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2rtwrg4pjxl2--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2klmb-6glqn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"333a2f1360dd6224035baf136a6281579fb567d23bfe26dee12ef543a72e7859","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want \u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-\u003cid\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-\u003cid\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["333a2f1360dd6224035baf136a6281579fb567d23bfe26dee12ef543a72e7859"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103519896854859776","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T16:19:59Z","started_at":"2026-09-25T16:19:59Z","completed_at":"2026-09-25T18:36:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T18:46:19.238286839Z","timing_metadata_checked_at":"2026-09-25T18:46:19.238286839Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T18:46:19.242699125Z","region_metadata_checked_at":"2026-09-25T18:46:19.242699125Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"194b32ef5888c901aadec1fe0f229081467e1037a965fd9c764b1c5b95bbe200","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103519896854859776","occurred_at":"2026-09-25T16:19:59Z","failed_at":"e2e","signature_id":"e625f6b2928df703a3e0c754e899a049523cbde56debbcd37376834e602b6278","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e30fe0\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-zw4cd4\" in resourcegroup=\"rg-np-version-upgrade-zw4cd4-2qsd7bvk6zfl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/995a0744-273b-428e-a7c0-5da145da2012\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/995a0744-273b-428e-a7c0-5da145da2012\",\n \"name\": \"995a0744-273b-428e-a7c0-5da145da2012\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T17:47:45.849510742Z\",\n \"endTime\": \"2026-09-25T18:06:46.935940984Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-zw4cd4-2qsd7bvk6zfl-np-upgrade-zw-63ddcd9b/providers/Microsoft.Compute/virtualMachines/p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-zw4cd4-2qsd7bvk6zfl-np-upgrade-zw-63ddcd9b/providers/Microsoft.Compute/virtualMachines/p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e30fe0\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-zw4cd4\" in resourcegroup=\"rg-np-version-upgrade-zw4cd4-2qsd7bvk6zfl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/995a0744-273b-428e-a7c0-5da145da2012\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/995a0744-273b-428e-a7c0-5da145da2012\",\n \"name\": \"995a0744-273b-428e-a7c0-5da145da2012\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T17:47:45.849510742Z\",\n \"endTime\": \"2026-09-25T18:06:46.935940984Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-zw4cd4-2qsd7bvk6zfl-np-upgrade-zw-63ddcd9b/providers/Microsoft.Compute/virtualMachines/p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-zw4cd4-2qsd7bvk6zfl-np-upgrade-zw-63ddcd9b/providers/Microsoft.Compute/virtualMachines/p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"96800d7c7de38f007fb6c1e0881939b8ef8d0c398e84b2e8ff15cc298d9bdcd5","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"752d6cc36815aed1b28a6ad7260aacf0e6af0885b17658028f416c067195a027","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103519896854859776","occurred_at":"2026-09-25T16:19:59Z","failed_at":"e2e","signature_id":"d94abe80bfe3331c4bae8c789b9cd27e17b79e88882a8166c69ed09fb5c17b41","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000feb180\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-jmvth8gmbtdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c689250c-de76-4270-9b21-5b67d697be13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c689250c-de76-4270-9b21-5b67d697be13\",\n \"name\": \"c689250c-de76-4270-9b21-5b67d697be13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T16:52:06.978905427Z\",\n \"endTime\": \"2026-09-25T17:11:15.434963979Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-jmvth8gmbtdp--managed/providers/Microsoft.Compute/virtualMachines/y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-jmvth8gmbtdp--managed/providers/Microsoft.Compute/virtualMachines/y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000feb180\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-jmvth8gmbtdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c689250c-de76-4270-9b21-5b67d697be13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c689250c-de76-4270-9b21-5b67d697be13\",\n \"name\": \"c689250c-de76-4270-9b21-5b67d697be13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T16:52:06.978905427Z\",\n \"endTime\": \"2026-09-25T17:11:15.434963979Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-jmvth8gmbtdp--managed/providers/Microsoft.Compute/virtualMachines/y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-jmvth8gmbtdp--managed/providers/Microsoft.Compute/virtualMachines/y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"96800d7c7de38f007fb6c1e0881939b8ef8d0c398e84b2e8ff15cc298d9bdcd5","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["96800d7c7de38f007fb6c1e0881939b8ef8d0c398e84b2e8ff15cc298d9bdcd5"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103344681369735168","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T04:43:44Z","started_at":"2026-09-25T04:43:44Z","completed_at":"2026-09-25T07:20:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T07:26:53.905148764Z","timing_metadata_checked_at":"2026-09-25T07:26:53.905148764Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T07:26:53.907919755Z","region_metadata_checked_at":"2026-09-25T07:26:53.907919755Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"5943e2da0a887cccda8bc216ba3a86640b2fb22d0ea66a8c812dc804eb1d0308","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103344681369735168","occurred_at":"2026-09-25T04:43:44Z","failed_at":"e2e","signature_id":"e3d755946357e33c2f103362fb47a38b0d13a5be02426a90a6d3deb944030095","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxn248k7 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-hwjzhdrfqdml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc000973dd0\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxn248k7 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-hwjzhdrfqdml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc000973dd0\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"88c34ce3eb4dd39781e8ddf92764a71a4749619137d6e0e581fc858207a4e653","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}},{"row_id":"d429e20bf6f8f867da754fb4118790bd1d4e9c481b035018d7b8865bfc567ab0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103344681369735168","occurred_at":"2026-09-25T04:43:44Z","failed_at":"e2e","signature_id":"d0a53dcf7d7bdcb9735560a86453bcca992d3fce7123e13092abd431e1b00745","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001140700\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-98gh8s6zjjg9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c466aed6-2164-42ab-a3c4-c5d9337db2b5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c466aed6-2164-42ab-a3c4-c5d9337db2b5\",\n \"name\": \"c466aed6-2164-42ab-a3c4-c5d9337db2b5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T05:20:30.850301491Z\",\n \"endTime\": \"2026-09-25T05:39:34.701925259Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v5v6d2f0y9a3z6e-np-1-r5nkw-flxjm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-98gh8s6zjjg9--managed/providers/Microsoft.Compute/virtualMachines/v5v6d2f0y9a3z6e-np-1-r5nkw-flxjm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v5v6d2f0y9a3z6e-np-1-r5nkw-flxjm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-98gh8s6zjjg9--managed/providers/Microsoft.Compute/virtualMachines/v5v6d2f0y9a3z6e-np-1-r5nkw-flxjm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001140700\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-98gh8s6zjjg9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c466aed6-2164-42ab-a3c4-c5d9337db2b5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c466aed6-2164-42ab-a3c4-c5d9337db2b5\",\n \"name\": \"c466aed6-2164-42ab-a3c4-c5d9337db2b5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T05:20:30.850301491Z\",\n \"endTime\": \"2026-09-25T05:39:34.701925259Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v5v6d2f0y9a3z6e-np-1-r5nkw-flxjm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-98gh8s6zjjg9--managed/providers/Microsoft.Compute/virtualMachines/v5v6d2f0y9a3z6e-np-1-r5nkw-flxjm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v5v6d2f0y9a3z6e-np-1-r5nkw-flxjm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-98gh8s6zjjg9--managed/providers/Microsoft.Compute/virtualMachines/v5v6d2f0y9a3z6e-np-1-r5nkw-flxjm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"96800d7c7de38f007fb6c1e0881939b8ef8d0c398e84b2e8ff15cc298d9bdcd5","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"de83d055e085ab402e87d7a22828af6c78b39eec45ae3bcad23d96c9c264b59e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103344681369735168","occurred_at":"2026-09-25T04:43:44Z","failed_at":"e2e","signature_id":"3f9b45040bd18db3decbda1404a3ed874d541ef97cff863c3152e40ed839bf48","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:214]: failed to cordon node d7p3x3q5p9p9x4z-cordon-np-7bdwq-kf6d6\nUnexpected error:\n \u003c*errors.StatusError | 0xc000b17180\u003e: \n Operation cannot be fulfilled on nodes \"d7p3x3q5p9p9x4z-cordon-np-7bdwq-kf6d6\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:214]: failed to cordon node d7p3x3q5p9p9x4z-cordon-np-7bdwq-kf6d6\nUnexpected error:\n \u003c*errors.StatusError | 0xc000b17180\u003e: \n Operation cannot be fulfilled on nodes \"d7p3x3q5p9p9x4z-cordon-np-7bdwq-kf6d6\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d6238e39baae339fded79308f27dee513cc7e737ebdf99b36269c7a4d445524","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0d6238e39baae339fded79308f27dee513cc7e737ebdf99b36269c7a4d445524","88c34ce3eb4dd39781e8ddf92764a71a4749619137d6e0e581fc858207a4e653","96800d7c7de38f007fb6c1e0881939b8ef8d0c398e84b2e8ff15cc298d9bdcd5"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2103303533246812160","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-25T02:00:14Z","started_at":"2026-09-25T02:00:14Z","completed_at":"2026-09-25T04:12:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T04:19:10.413978386Z","timing_metadata_checked_at":"2026-09-25T04:19:10.413978386Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T04:19:10.417654574Z","region_metadata_checked_at":"2026-09-25T04:19:10.417654574Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b1b6853a1134e1a6f9b885ae49866a8b3fac365f8773b854962f2a8dcdcdbd9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2103303533246812160","occurred_at":"2026-09-25T02:00:14Z","failed_at":"e2e","signature_id":"889564d15d770295c243009ba9604dcfbcd6afb5d6a4b37657d92965a6c4aa98","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox4lqltr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-gvcrd9xg4vzv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0008ace00\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox4lqltr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-gvcrd9xg4vzv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0008ace00\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"88c34ce3eb4dd39781e8ddf92764a71a4749619137d6e0e581fc858207a4e653","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["88c34ce3eb4dd39781e8ddf92764a71a4749619137d6e0e581fc858207a4e653"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}