{"meta":{"start_date":"2026-09-26","end_date":"2026-09-26","start_at":"2026-09-26T00:00:00Z","end_at":"2026-09-27T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T07:59:49Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":76,"failed_runs":75,"raw_occurrences":102,"matched_occurrences":101,"runs_affected":74},"rows":[{"environment":"dev","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":60,"runs_affected":60,"failed_runs":60,"run_impact_percent":78.94736842105263,"anchor_occurrences":60,"after_last_push_count":47,"trend_counts":[0,0,0,0,11,43,60],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":114,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":60}],"full_error_samples":["time=2026-09-26T14:49:06.836Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:49:08.412Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:49:09.638Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=6089940dde784d763a773e4c9004101f74c31b1e2eea1a61564da81e9a0036e5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5653504%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6089940dde784d763a773e4c9004101f74c31b1e2eea1a61564da81e9a0036e5\ntime=2026-09-26T14:52:40.545Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5653504/providers/Microsoft.Resources/deployments/6089940dde784d763a773e4c9004101f74c31b1e2eea1a61564da81e9a0036e5/operationStatuses/08584111727364954260\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5653504/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5653504/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5653504\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 81aead43-b997-4ca0-92b1-5f890a98ae35:9/26/2026 2:50:21 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T14:54:40.554Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:54:41.206Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:54:42.164Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9881912e12f161c343a185ab325eab0f9959440aad9a04d75fdbf5669aaa2bef portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5653504%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9881912e12f161c343a185ab325eab0f9959440aad9a04d75fdbf5669aaa2bef\ntime=2026-09-26T15:10:45.962Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5653504/providers/Microsoft.Resources/deployments/9881912e12f161c343a185ab325eab0f9959440aad9a04d75fdbf5669aaa2bef/operationStatuses/08584111724039322810\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5653504/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5653504/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5653504\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 7053750e-7976-4092-b5c9-05af8bee4bc0:9/26/2026 3:10:24 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T15:12:45.966Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T15:12:46.574Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T15:12:47.251Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5c70e5fdb4253270876a8cf87e4ad360a842c5c82a499a0efaf8267c12be42a2 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5653504%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5c70e5fdb4253270876a8cf87e4ad360a842c5c82a499a0efaf8267c12be42a2\ntime=2026-09-26T15:19:06.585Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-26T14:49:01.802Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:49:03.910Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:49:05.531Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9ec5d968c4a2abbfa09f2d13ea926004d6d393845d4e1c03095043f158076d84 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6813568%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9ec5d968c4a2abbfa09f2d13ea926004d6d393845d4e1c03095043f158076d84\ntime=2026-09-26T14:52:06.712Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6813568/providers/Microsoft.Resources/deployments/9ec5d968c4a2abbfa09f2d13ea926004d6d393845d4e1c03095043f158076d84/operationStatuses/08584111727409288089\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6813568/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6813568/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6813568\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 3938f6fe-2589-4619-be28-fc66822767aa:9/26/2026 2:50:13 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T14:54:06.713Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:54:07.745Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:54:09.171Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8afa4a83c65198c5a6152d3ce561b308250ec90817107c38ff6b5a156799c6da portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6813568%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8afa4a83c65198c5a6152d3ce561b308250ec90817107c38ff6b5a156799c6da\ntime=2026-09-26T15:09:44.252Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6813568/providers/Microsoft.Resources/deployments/8afa4a83c65198c5a6152d3ce561b308250ec90817107c38ff6b5a156799c6da/operationStatuses/08584111724372557912\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6813568/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6813568/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6813568\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 381e25a6-705f-4967-8b86-0e72fd08d4dc:9/26/2026 3:09:17 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T15:11:44.253Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T15:11:45.072Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T15:11:46.506Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f8da50925da962c46c0feb369b1ccdd3215b84037bcc35598bc1f3f1121f0d5d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6813568%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff8da50925da962c46c0feb369b1ccdd3215b84037bcc35598bc1f3f1121f0d5d\ntime=2026-09-26T15:19:01.486Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-26T14:43:15.259Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:43:16.912Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:43:17.849Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=553cb6c6b3fded1accbd2852e1223b9f885add89fe9c1ca9b9ab9fd387804af3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1104128%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F553cb6c6b3fded1accbd2852e1223b9f885add89fe9c1ca9b9ab9fd387804af3\ntime=2026-09-26T14:47:48.727Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1104128/providers/Microsoft.Resources/deployments/553cb6c6b3fded1accbd2852e1223b9f885add89fe9c1ca9b9ab9fd387804af3/operationStatuses/08584111730881613482\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1104128/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1104128/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1104128\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 5fd50c6a-1737-4597-a037-368930da6cc9:9/26/2026 2:44:27 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T14:49:48.731Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:49:49.490Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:49:50.425Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=99ebb22d91f4358f020049c1b68a81eba2ea522f9bacb97496c9a72de75bf919 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1104128%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F99ebb22d91f4358f020049c1b68a81eba2ea522f9bacb97496c9a72de75bf919\ntime=2026-09-26T15:05:54.057Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1104128/providers/Microsoft.Resources/deployments/99ebb22d91f4358f020049c1b68a81eba2ea522f9bacb97496c9a72de75bf919/operationStatuses/08584111726955290959\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1104128/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1104128/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1104128\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 169a3df3-f8a3-4c96-9074-128137b133d8:9/26/2026 3:05:31 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T15:07:54.062Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T15:07:54.711Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T15:07:55.697Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9b984337a990465b3d1ba4b559dcc1260b91bb6b32219cd0080e1d7d34f722df portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1104128%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9b984337a990465b3d1ba4b559dcc1260b91bb6b32219cd0080e1d7d34f722df\ntime=2026-09-26T15:13:14.833Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103852034485653504","occurred_at":"2026-09-26T14:19:35Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851390936813568","occurred_at":"2026-09-26T14:17:13Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851378941104128","occurred_at":"2026-09-26T14:17:11Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851381092782080","occurred_at":"2026-09-26T14:17:11Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7188,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851372238606336","occurred_at":"2026-09-26T14:17:09Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851355876626432","occurred_at":"2026-09-26T14:17:05Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851334296932352","occurred_at":"2026-09-26T14:17:00Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851316445974528","occurred_at":"2026-09-26T14:16:56Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103848603104579584","occurred_at":"2026-09-26T14:06:07Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103848790996815872","occurred_at":"2026-09-26T13:01:11Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103848898379386880","occurred_at":"2026-09-26T13:00:47Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103790396751482880","occurred_at":"2026-09-26T10:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103789910530985984","occurred_at":"2026-09-26T10:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103786893190631424","occurred_at":"2026-09-26T10:00:44Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6846,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103785005913214976","occurred_at":"2026-09-26T09:53:26Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103775301015179264","occurred_at":"2026-09-26T09:14:52Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103774808339648512","occurred_at":"2026-09-26T09:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103764124197457920","occurred_at":"2026-09-26T08:30:26Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103760196256927744","occurred_at":"2026-09-26T08:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103759712255217664","occurred_at":"2026-09-26T08:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103745097093877760","occurred_at":"2026-09-26T07:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103744610089046016","occurred_at":"2026-09-26T07:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103733651282857984","occurred_at":"2026-09-26T06:29:22Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103730519727411200","occurred_at":"2026-09-26T06:16:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103729999365279744","occurred_at":"2026-09-26T06:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103729510443651072","occurred_at":"2026-09-26T06:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103714900751683584","occurred_at":"2026-09-26T05:14:52Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103714409951006720","occurred_at":"2026-09-26T05:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103705397104742400","occurred_at":"2026-09-26T04:37:06Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103701911558426624","occurred_at":"2026-09-26T04:23:15Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103699803396378624","occurred_at":"2026-09-26T04:14:52Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103699317209436160","occurred_at":"2026-09-26T04:12:56Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103685753023238144","occurred_at":"2026-09-26T03:19:02Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103684698914951168","occurred_at":"2026-09-26T03:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103684219090767872","occurred_at":"2026-09-26T03:12:57Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670738761289728","occurred_at":"2026-09-26T02:19:23Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670723087175680","occurred_at":"2026-09-26T02:19:19Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670718657990656","occurred_at":"2026-09-26T02:19:18Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670704342831104","occurred_at":"2026-09-26T02:19:14Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670699393552384","occurred_at":"2026-09-26T02:19:13Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7168,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670647581315072","occurred_at":"2026-09-26T02:19:01Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670621719236608","occurred_at":"2026-09-26T02:18:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103667365953671168","occurred_at":"2026-09-26T02:05:58Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103667354889097216","occurred_at":"2026-09-26T02:05:56Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103656984816390144","occurred_at":"2026-09-26T01:24:43Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103655450791645184","occurred_at":"2026-09-26T01:18:38Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103655428259844096","occurred_at":"2026-09-26T01:18:32Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7188,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103654924834312192","occurred_at":"2026-09-26T01:16:32Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103650148490940416","occurred_at":"2026-09-26T00:57:33Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6775,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641099342974976","occurred_at":"2026-09-26T00:21:36Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641079956901888","occurred_at":"2026-09-26T00:21:31Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641065637548032","occurred_at":"2026-09-26T00:21:28Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641066266693632","occurred_at":"2026-09-26T00:21:28Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7168,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641060163981312","occurred_at":"2026-09-26T00:21:27Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641026831847424","occurred_at":"2026-09-26T00:21:19Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641014462844928","occurred_at":"2026-09-26T00:21:16Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641015683387392","occurred_at":"2026-09-26T00:21:16Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7176,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103640998813896704","occurred_at":"2026-09-26T00:21:12Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103639551695130624","occurred_at":"2026-09-26T00:15:27Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103639238770692096","occurred_at":"2026-09-26T00:14:12Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7181,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":6.578947368421052,"anchor_occurrences":5,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,6,5],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":11,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:30:46Z\n Ended: 2026-09-27T00:51:51Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j9219200-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t317no0qn2vco31uu1nj119agt86hub-agg-cluster\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j9219200-mgmt-1\nNamespace: ocm-arohcpci01-2t317no0qn2vco31uu1nj119agt86hub-agg-cluster","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T20:33:18Z\n Ended: 2026-09-26T20:58:20Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j6283520-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t2tnffjjbth0k2idnuevp5detncdatr-cilium-cluster\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j6283520-mgmt-2\nNamespace: ocm-arohcpci01-2t2tnffjjbth0k2idnuevp5detncdatr-cilium-cluster","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T17:32:15Z\n Ended: 2026-09-26T17:39:15Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j0431360-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t2qrbqcgq901aevacbo6vkcduiahunj-acr-pull-create\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0431360-mgmt-2\nNamespace: ocm-arohcpci01-2t2qrbqcgq901aevacbo6vkcduiahunj-acr-pull-create"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7194/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103990456999219200","occurred_at":"2026-09-26T23:29:49Z","signature_id":"6566df96cb73a9adffc87a980a6110cb9f6cb7fac4d1aef176d37f92cb479ed6","pr_number":7194,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103929857506283520","occurred_at":"2026-09-26T19:29:01Z","signature_id":"b687d77087ab0324f2e1d0aa267ff4136bba5841aa7af068e1fef75e22687772","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881473420431360","occurred_at":"2026-09-26T16:16:46Z","signature_id":"e6f461584d644efe4659caaf9224f6ef200ce34ae3721cb24a9c26cca546b081","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881450792161280","occurred_at":"2026-09-26T16:16:40Z","signature_id":"028b8b3f149367175ed04cdc868d03ee9feeb1166c251ae199607b3d027a8634","pr_number":7188,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881428969197568","occurred_at":"2026-09-26T16:16:35Z","signature_id":"29c97a1bc4096ae773b9213705339dee20d316902decd611365f2238cc2051e0","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":6.578947368421052,"anchor_occurrences":5,"after_last_push_count":65,"trend_counts":[5,20,21,22,24,30,5],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":159,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:18:12Z\n Ended: 2026-09-27T00:27:15Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6084992-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6084992-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T17:44:26Z\n Ended: 2026-09-26T17:50:28Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6814080-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6814080-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T17:44:51Z\n Ended: 2026-09-26T17:50:50Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7663616-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7663616-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"ab36f01ba1f6dd47c0278082b2e3a50d4690b691b0c683c8e73be1e12cca3d69","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881469456814080","occurred_at":"2026-09-26T16:16:45Z","signature_id":"342e9a20297ec559151d137a61844f3c9a0a2d3d9dd6a479df3a39341749c146","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881459847663616","occurred_at":"2026-09-26T16:16:42Z","signature_id":"ebafac72ebe108727ebff5313e306f4f9049eb95d8f8e7da4ad87e9d88b60349","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881428969197568","occurred_at":"2026-09-26T16:16:35Z","signature_id":"8e031d136666491c65ac1f03120c0173819d20c1c3adaf01d791c67e1aef0c7c","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881405791473664","occurred_at":"2026-09-26T16:16:29Z","signature_id":"c4223b7b494a6dc2601c8c97e4b1d46697163b469eaf658557332ddc1cec8106","pr_number":7068,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":5.2631578947368425,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,5,4],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","occurrences":4}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:33:47Z\n Ended: 2026-09-27T00:52:51Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j9219200-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t317no0qn2vco31uu1nj119agt86hub-agg-cluster\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j9219200-mgmt-1\nNamespace: ocm-arohcpci01-2t317no0qn2vco31uu1nj119agt86hub-agg-cluster","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T20:37:19Z\n Ended: 2026-09-26T21:23:32Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j6283520-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t2tnffjjbth0k2idnuevp5detncdatr-cilium-cluster\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j6283520-mgmt-2\nNamespace: ocm-arohcpci01-2t2tnffjjbth0k2idnuevp5detncdatr-cilium-cluster","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T17:35:15Z\n Ended: 2026-09-26T18:04:16Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j0431360-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t2qrbqcgq901aevacbo6vkcduiahunj-acr-pull-create\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0431360-mgmt-2\nNamespace: ocm-arohcpci01-2t2qrbqcgq901aevacbo6vkcduiahunj-acr-pull-create"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7194/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103990456999219200","occurred_at":"2026-09-26T23:29:49Z","signature_id":"118b9a2c18bb76c55af60628010650bc62252eec1df7945fcd3c5d3154495df9","pr_number":7194,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103929857506283520","occurred_at":"2026-09-26T19:29:01Z","signature_id":"9ca4bc2deb7a2ca544935fb63e821804c0dc6f87fddd7c30fc0e84f823cb1c81","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881473420431360","occurred_at":"2026-09-26T16:16:46Z","signature_id":"7035fca2cdc8aba021f22051d3cd5b3870f8b1d6ff824a3d51afe42c3c74498a","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881428969197568","occurred_at":"2026-09-26T16:16:35Z","signature_id":"8fb687948b9e439949059b6fd34880b83b1afcd34a8ef97d1f528299dcd7501c","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":5.2631578947368425,"anchor_occurrences":4,"after_last_push_count":6,"trend_counts":[0,0,2,0,1,0,4],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":17,"prior_last_seen_at":"2026-09-19T02:41:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103929841903472640","occurred_at":"2026-09-26T19:28:58Z","signature_id":"817dbbd2f4e987cf783899d57ba9ab565857a252a91d7741c25fbfc506aff5e8","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927280047755264","occurred_at":"2026-09-26T19:18:47Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881469456814080","occurred_at":"2026-09-26T16:16:45Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":3,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,5,3],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T23:57:44Z\n Ended: 2026-09-27T00:21:42Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j6084992-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t30min1009r4686mdn5h02sl9p5nb55-r4w4c9x5f5e5h4o\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j6084992-mgmt-2\nNamespace: ocm-arohcpci01-2t30min1009r4686mdn5h02sl9p5nb55-r4w4c9x5f5e5h4o","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T22:38:39Z\n Ended: 2026-09-26T23:07:40Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j0737536-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t2viddtv02l63m6ul7m30kp2hdmu31l-agg-cluster\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0737536-mgmt-1\nNamespace: ocm-arohcpci01-2t2viddtv02l63m6ul7m30kp2hdmu31l-agg-cluster","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T20:32:56Z\n Ended: 2026-09-26T20:58:54Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j2450304-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t2tneh2r23glrlaq41sk5licb45u3pf-l2y8b5i4r2k8w3t\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2450304-mgmt-2\nNamespace: ocm-arohcpci01-2t2tneh2r23glrlaq41sk5licb45u3pf-l2y8b5i4r2k8w3t\n\nFiring 2:\n State: Resolved\n Started: 2026-09-26T20:34:58Z\n Ended: 2026-09-26T20:56:56Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j2450304-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t2tna269g6d79a0hm4q37mmv6lq3q16-private-kas\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2450304-mgmt-2\nNamespace: ocm-arohcpci01-2t2tna269g6d79a0hm4q37mmv6lq3q16-private-kas"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"3a14be93314f64bd1aaf34287e4e0adef28fca1a41e77dc2ba5a7194956ee79f","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103965959260737536","occurred_at":"2026-09-26T21:52:29Z","signature_id":"11ba818ac5dd1ff347068d274c7fa5d21dba08c20baa52b6b0f46bbac6c58411","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"5afa5c76e03a3b0c3b55adc55370d70f4ab3538b83cc562c8d049fb3ad18fe8a","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,4,3],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:00:39Z\n Ended: 2026-09-27T00:46:41Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j6084992-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t30min1009r4686mdn5h02sl9p5nb55-r4w4c9x5f5e5h4o\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j6084992-mgmt-2\nNamespace: ocm-arohcpci01-2t30min1009r4686mdn5h02sl9p5nb55-r4w4c9x5f5e5h4o","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T22:41:41Z\n Ended: 2026-09-26T23:32:41Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j0737536-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t2viddtv02l63m6ul7m30kp2hdmu31l-agg-cluster\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0737536-mgmt-1\nNamespace: ocm-arohcpci01-2t2viddtv02l63m6ul7m30kp2hdmu31l-agg-cluster","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T20:35:56Z\n Ended: 2026-09-26T21:23:56Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j2450304-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t2tneh2r23glrlaq41sk5licb45u3pf-l2y8b5i4r2k8w3t\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2450304-mgmt-2\nNamespace: ocm-arohcpci01-2t2tneh2r23glrlaq41sk5licb45u3pf-l2y8b5i4r2k8w3t\n\nFiring 2:\n State: Resolved\n Started: 2026-09-26T20:37:57Z\n Ended: 2026-09-26T21:21:56Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j2450304-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t2tna269g6d79a0hm4q37mmv6lq3q16-private-kas\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2450304-mgmt-2\nNamespace: ocm-arohcpci01-2t2tna269g6d79a0hm4q37mmv6lq3q16-private-kas"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"878ce001cd49a7ed748478aef200a126d7f5d5a033ae0db1aa1de44d5ddcc877","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103965959260737536","occurred_at":"2026-09-26T21:52:29Z","signature_id":"1d021545a8594032e7b33f8a228af4da144b809174ff6174498d77fa92101d7c","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"983ca4f9ae3ed26808cbee2ea2a4466c45d7f3a39a19058178e4a59512e41739","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Network","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,2,2],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wx6szx\" for cluster \"cluster-candidate-4-20-wx6szx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b82600\u003e: \n failed to create NodePool np-wx6szx: failed waiting for nodepool=\"np-wx6szx\" for cluster \"cluster-candidate-4-20-wx6szx\" in resourcegroup=\"rg-candidate-4-20-r2kvmlfvs4fd\" to finish creating: GET https://rp.j0737536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8ae21a1-086f-493e-86bc-e120b31e75a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8ae21a1-086f-493e-86bc-e120b31e75a1\",\n \"name\": \"a8ae21a1-086f-493e-86bc-e120b31e75a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:38:07.749745969Z\",\n \"endTime\": \"2026-09-26T22:57:10.827139148Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-20-r2kvmlfvs4fd-candidate-wx6szx--managed/x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-r2kvmlfvs4fd-candidate-wx6szx--managed/providers/Microsoft.Network/networkInterfaces/x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wx6szx\" for cluster \"cluster-candidate-4-20-wx6szx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b82600\u003e: \n failed to create NodePool np-wx6szx: failed waiting for nodepool=\"np-wx6szx\" for cluster \"cluster-candidate-4-20-wx6szx\" in resourcegroup=\"rg-candidate-4-20-r2kvmlfvs4fd\" to finish creating: GET https://rp.j0737536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8ae21a1-086f-493e-86bc-e120b31e75a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8ae21a1-086f-493e-86bc-e120b31e75a1\",\n \"name\": \"a8ae21a1-086f-493e-86bc-e120b31e75a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:38:07.749745969Z\",\n \"endTime\": \"2026-09-26T22:57:10.827139148Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-20-r2kvmlfvs4fd-candidate-wx6szx--managed/x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-r2kvmlfvs4fd-candidate-wx6szx--managed/providers/Microsoft.Network/networkInterfaces/x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vcbnqz\" for cluster \"cluster-candidate-4-20-vcbnqz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe65e0\u003e: \n failed to create NodePool np-vcbnqz: failed waiting for nodepool=\"np-vcbnqz\" for cluster \"cluster-candidate-4-20-vcbnqz\" in resourcegroup=\"rg-candidate-4-20-frjpqt6l6dlc\" to finish creating: GET https://rp.j7663616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8abd2b48-24de-4fb1-bba7-e70679eac440\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8abd2b48-24de-4fb1-bba7-e70679eac440\",\n \"name\": \"8abd2b48-24de-4fb1-bba7-e70679eac440\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T17:19:57.699169962Z\",\n \"endTime\": \"2026-09-26T17:38:57.270667313Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i2k9x4t6q6r7f1s-np-vcbnqz-qnsq9-j9kx5: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-20-frjpqt6l6dlc-candidate-vcbnqz--managed/i2k9x4t6q6r7f1s-np-vcbnqz-qnsq9-j9kx5-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-frjpqt6l6dlc-candidate-vcbnqz--managed/providers/Microsoft.Network/networkInterfaces/i2k9x4t6q6r7f1s-np-vcbnqz-qnsq9-j9kx5-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '12eafc4e-f869-4b62-8198-816b7d6d0876' exceeded. Please try again after '10' ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vcbnqz\" for cluster \"cluster-candidate-4-20-vcbnqz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe65e0\u003e: \n failed to create NodePool np-vcbnqz: failed waiting for nodepool=\"np-vcbnqz\" for cluster \"cluster-candidate-4-20-vcbnqz\" in resourcegroup=\"rg-candidate-4-20-frjpqt6l6dlc\" to finish creating: GET https://rp.j7663616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8abd2b48-24de-4fb1-bba7-e70679eac440\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8abd2b48-24de-4fb1-bba7-e70679eac440\",\n \"name\": \"8abd2b48-24de-4fb1-bba7-e70679eac440\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T17:19:57.699169962Z\",\n \"endTime\": \"2026-09-26T17:38:57.270667313Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i2k9x4t6q6r7f1s-np-vcbnqz-qnsq9-j9kx5: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-20-frjpqt6l6dlc-candidate-vcbnqz--managed/i2k9x4t6q6r7f1s-np-vcbnqz-qnsq9-j9kx5-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-frjpqt6l6dlc-candidate-vcbnqz--managed/providers/Microsoft.Network/networkInterfaces/i2k9x4t6q6r7f1s-np-vcbnqz-qnsq9-j9kx5-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '12eafc4e-f869-4b62-8198-816b7d6d0876' exceeded. Please try again after '10' ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103965959260737536","occurred_at":"2026-09-26T21:52:29Z","signature_id":"8a579545eb8ffc7ce43e3545cb26540ccd7358fcf1514bbca20793979fbd673e","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881459847663616","occurred_at":"2026-09-26T16:16:42Z","signature_id":"043f03e78369abc3dac495551386e55ba539c998c6f45575a66419b88c93eda0","pr_number":7164,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[1,1,3,2,4,1,2],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":20,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":2}],"full_error_samples":["alert fired 4 time(s) (1 unknown, 3 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:23:15Z\n Ended: 2026-09-27T00:29:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t30n01ijoaem1ekng72tj4pcrtjldgt-u0g5w9w4b6k9u1r\", pod=\"kube-apiserver-6996784dc-8vzfd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t30n01ijoaem1ekng72tj4pcrtjldgt-u0g5w9w4b6k9u1r/kube-apiserver-6996784dc-8vzfd has been in a non-ready state for longer than 5 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T20:54:34Z\n Ended: 2026-09-26T21:00:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6283520-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t2u1irnugb4blbq80ksmoo4fkldqnt8-oidc-wi-cluster\", pod=\"kube-controller-manager-65c84bdd67-wnnzz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t2u1irnugb4blbq80ksmoo4fkldqnt8-oidc-wi-cluster/kube-controller-manager-65c84bdd67-wnnzz has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-26T21:02:36Z\n Ended: 2026-09-26T21:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6283520-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t2tnr9frt52ianj6sih8n7ki9jvmnj1-v9n4d3d5t5m9z0v\", pod=\"kube-apiserver-797f569f98-n962n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t2tnr9frt52ianj6sih8n7ki9jvmnj1-v9n4d3d5t5m9z0v/kube-apiserver-797f569f98-n962n has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"f6e1a0738cb75e810c1d760e4593e47ecd222a26626a4f413d3bafc5f44c9529","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103929857506283520","occurred_at":"2026-09-26T19:29:01Z","signature_id":"a5c5601829d7dc2e22894167e287975c02c8a19694717b357e8b26e5dffcb3d2","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":2,"after_last_push_count":3,"also_in":["STG"],"trend_counts":[0,0,0,0,0,6,2],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-9cp272\" for cluster \"cluster-candidate-4-23-9cp272\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019bf20\u003e: \n failed to create NodePool np-9cp272: failed waiting for nodepool=\"np-9cp272\" for cluster \"cluster-candidate-4-23-9cp272\" in resourcegroup=\"rg-candidate-4-23-t5nrns5dgzjz\" to finish creating: GET https://rp.j6084992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92540fc4-c95a-4a5d-a847-5375c72243a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92540fc4-c95a-4a5d-a847-5375c72243a8\",\n \"name\": \"92540fc4-c95a-4a5d-a847-5375c72243a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-27T00:01:58.819547057Z\",\n \"endTime\": \"2026-09-27T00:21:07.730201043Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t5nrns5dgzjz-candidate-9cp272--managed/providers/Microsoft.Compute/virtualMachines/l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t5nrns5dgzjz-candidate-9cp272--managed/providers/Microsoft.Compute/virtualMachines/l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-9cp272\" for cluster \"cluster-candidate-4-23-9cp272\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019bf20\u003e: \n failed to create NodePool np-9cp272: failed waiting for nodepool=\"np-9cp272\" for cluster \"cluster-candidate-4-23-9cp272\" in resourcegroup=\"rg-candidate-4-23-t5nrns5dgzjz\" to finish creating: GET https://rp.j6084992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92540fc4-c95a-4a5d-a847-5375c72243a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92540fc4-c95a-4a5d-a847-5375c72243a8\",\n \"name\": \"92540fc4-c95a-4a5d-a847-5375c72243a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-27T00:01:58.819547057Z\",\n \"endTime\": \"2026-09-27T00:21:07.730201043Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t5nrns5dgzjz-candidate-9cp272--managed/providers/Microsoft.Compute/virtualMachines/l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t5nrns5dgzjz-candidate-9cp272--managed/providers/Microsoft.Compute/virtualMachines/l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d2f360\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-9sjsgl\" in resourcegroup=\"rg-np-version-upgrade-9sjsgl-z2gscsmxlxzq\" to finish creating: GET https://rp.j6084992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ef95de3-9fbf-4193-8079-b1158861e584\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ef95de3-9fbf-4193-8079-b1158861e584\",\n \"name\": \"2ef95de3-9fbf-4193-8079-b1158861e584\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:57:59.240971764Z\",\n \"endTime\": \"2026-09-27T00:17:06.977144117Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9sjsgl-z2gscsmxlxzq-np-upgrade-9s-96d5bba1/providers/Microsoft.Compute/virtualMachines/w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9sjsgl-z2gscsmxlxzq-np-upgrade-9s-96d5bba1/providers/Microsoft.Compute/virtualMachines/w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d2f360\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-9sjsgl\" in resourcegroup=\"rg-np-version-upgrade-9sjsgl-z2gscsmxlxzq\" to finish creating: GET https://rp.j6084992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ef95de3-9fbf-4193-8079-b1158861e584\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ef95de3-9fbf-4193-8079-b1158861e584\",\n \"name\": \"2ef95de3-9fbf-4193-8079-b1158861e584\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:57:59.240971764Z\",\n \"endTime\": \"2026-09-27T00:17:06.977144117Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9sjsgl-z2gscsmxlxzq-np-upgrade-9s-96d5bba1/providers/Microsoft.Compute/virtualMachines/w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9sjsgl-z2gscsmxlxzq-np-upgrade-9s-96d5bba1/providers/Microsoft.Compute/virtualMachines/w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"3b383ee2285332a194dbad5c697b0305b25dd8206e97711a9b282832753b74b4","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"d103024f5017fdcabaff08262102c5247078ca8f9b984031104c706e84e3556e","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f78894a0f745284e7b454f80e662337a874e257b6590d707b4a761a42e23b9f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b86200\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9464kjvgqp5t\" to finish creating: GET https://rp.j0737536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\",\n \"name\": \"34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:26:42.363515696Z\",\n \"endTime\": \"2026-09-26T22:55:52.65701083Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, redhat-operators-catalog, certified-operators-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b86200\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9464kjvgqp5t\" to finish creating: GET https://rp.j0737536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\",\n \"name\": \"34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:26:42.363515696Z\",\n \"endTime\": \"2026-09-26T22:55:52.65701083Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, redhat-operators-catalog, certified-operators-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103965959260737536","occurred_at":"2026-09-26T21:52:29Z","signature_id":"214f1ab4117af1368e93bfb8d87ddb3947a25a6ea700ad87128366123ae0719e","pr_number":7185,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,2,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102e380\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-2x6vlrpgq2bh\" to finish creating: GET https://rp.j6084992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47c0d2f0-4e66-4d5c-8e28-e145397d1b94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47c0d2f0-4e66-4d5c-8e28-e145397d1b94\",\n \"name\": \"47c0d2f0-4e66-4d5c-8e28-e145397d1b94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:46:18.825261842Z\",\n \"endTime\": \"2026-09-27T00:15:27.197366694Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102e380\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-2x6vlrpgq2bh\" to finish creating: GET https://rp.j6084992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47c0d2f0-4e66-4d5c-8e28-e145397d1b94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47c0d2f0-4e66-4d5c-8e28-e145397d1b94\",\n \"name\": \"47c0d2f0-4e66-4d5c-8e28-e145397d1b94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:46:18.825261842Z\",\n \"endTime\": \"2026-09-27T00:15:27.197366694Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"9d0706e3969af7c111fcca838b1e51305acfc48ec2465de31927959306c28e66","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-18T15:17:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0020ae090\u003e: \n resource group \"fips-enabled-k6r2vqf8b9xq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-k6r2vqf8b9xq\" to finish deleting: GET https://rp.j2450304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50efbb08-7b80-41ad-a7a9-a51f5231b37f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50efbb08-7b80-41ad-a7a9-a51f5231b37f\",\n \"name\": \"50efbb08-7b80-41ad-a7a9-a51f5231b37f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T20:41:31.45560964Z\",\n \"endTime\": \"2026-09-26T21:05:35.034630666Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2tmvnf0viulta1u93uf2mlthhfuvsu still exists (deletion dispatched at 2026-09-26T20:41:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0020ae090\u003e: \n resource group \"fips-enabled-k6r2vqf8b9xq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-k6r2vqf8b9xq\" to finish deleting: GET https://rp.j2450304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50efbb08-7b80-41ad-a7a9-a51f5231b37f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50efbb08-7b80-41ad-a7a9-a51f5231b37f\",\n \"name\": \"50efbb08-7b80-41ad-a7a9-a51f5231b37f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T20:41:31.45560964Z\",\n \"endTime\": \"2026-09-26T21:05:35.034630666Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2tmvnf0viulta1u93uf2mlthhfuvsu still exists (deletion dispatched at 2026-09-26T20:41:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"aa958e32e1c37b11d08d6ef5dd635ad8ce90eced3b3170eac46f774a812933cc","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-qqprr26gtjgf\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qqprr26gtjgf, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":12,"prior_last_seen_at":"2026-09-18T18:24:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:209]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002608b40\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-qqprr26gtjgf\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qqprr26gtjgf, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:209]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002608b40\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-qqprr26gtjgf\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qqprr26gtjgf, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927280047755264","occurred_at":"2026-09-26T19:18:47Z","signature_id":"e3b07109b49bcafaeae2aeacce0e01aa2904db04b4015d601de9455d55fb2226","pr_number":7164,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":13,"trend_counts":[3,2,3,0,0,2,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":28,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-26T21:24:27Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2450304-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2450304/providers/microsoft.monitor/accounts/hcps-j2450304\", namespace=\"ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-q9544\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-26T21:24:27Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2450304-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2450304/providers/microsoft.monitor/accounts/hcps-j2450304\", namespace=\"ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-q9544\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-26T21:27:26Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2450304-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2450304/providers/microsoft.monitor/accounts/hcps-j2450304\", namespace=\"ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-q9544\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-26T21:27:27Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2450304-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2450304/providers/microsoft.monitor/accounts/hcps-j2450304\", namespace=\"ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-q9544\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"59859e98e366b5af518d04f59b67247bf6559ad517e1190c64eba0f09c96ceae","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":10,"trend_counts":[1,2,2,1,0,1,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":15,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:09:50Z\n Ended: 2026-09-27T00:37:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/hcps-j6084992\", namespace=\"ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"bb068423-1854-49c8-a132-0d10a6392326\"\n Description: Pod ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-27T00:09:50Z\n Ended: 2026-09-27T00:37:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/hcps-j6084992\", namespace=\"ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"bb068423-1854-49c8-a132-0d10a6392326\"\n Description: Pod ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"0e8d71bcb4b8b8f14ef36438303ce153649aa4d8d3bd9d130dfcadf1c764fbd1","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":51,"trend_counts":[4,2,2,2,4,6,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":101,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T17:41:16Z\n Ended: 2026-09-26T17:47:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9197568-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t2qtvdnlaalvsj2ub667na9fqboc359-ephemeral-disk\", pod=\"ignition-server-7fd4ccdc4c-psn2m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t2qtvdnlaalvsj2ub667na9fqboc359-ephemeral-disk/ignition-server-7fd4ccdc4c-psn2m has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881428969197568","occurred_at":"2026-09-26T16:16:35Z","signature_id":"4887464f22fd02cf878c19ad54c98e5ffd7550a5bce16e494d595bda58002c06","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":17,"trend_counts":[4,4,3,2,1,3,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":29,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:06:56Z\n Ended: 2026-09-27T00:37:55Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/hcps-j6084992\", namespace=\"ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z7blf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-27T00:06:56Z\n Ended: 2026-09-27T00:36:52Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/hcps-j6084992\", namespace=\"ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z7blf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"0dfb7950c4b9435e9a2c892dc201219aa13f6c87439b016889682cade7a522c2","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":10,"trend_counts":[2,3,2,2,1,1,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":17,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:21:19Z\n Ended: 2026-09-27T00:27:16Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/services-j6084992\", namespace=\"klusterlet-2t30mivbmcgkot8vhp8oa13ibv1vnaut\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z7blf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t30mivbmcgkot8vhp8oa13ibv1vnaut/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"a2eb7618d1e7a8cd04863ecb99aab31fa66480f2629953e2c4928c2aba4d4ae5","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-19T20:01:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881405791473664","occurred_at":"2026-09-26T16:16:29Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":7068,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f76baa3d4767ea148b5500ace79ced7c3f0e0092f22b03bde161fd5f0c1796ae","failure_pattern":"run command \"e2e-runcommand-1790458991773693237\" failed with exit code 28 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1790458991773693237\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 608.861s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n \u003c*errors.errorString | 0xc00107dc50\u003e: \n run command \"e2e-runcommand-1790458991773693237\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 608.861s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n \u003c*errors.errorString | 0xc00107dc50\u003e: \n run command \"e2e-runcommand-1790458991773693237\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103942172993654784","occurred_at":"2026-09-26T20:17:58Z","signature_id":"4eac6d1d6c768c796dcee39de9244b0faf13f00b731ed7de804c181901f6e831","pr_number":7068,"after_last_push_of_merged_pr":false}]}]},{"environment":"int","summary":{"total_runs":5,"failed_runs":1,"raw_occurrences":1,"matched_occurrences":1,"runs_affected":1},"rows":[{"environment":"int","failure_pattern_id":"b3f3e0a11bd87e1d792b79fd0f26a19cb61cf8deaa26da50a5b798fb45c47433","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001d427a0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2f88crxn77t5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b648d347-dd3d-4511-8fd6-84bc235918eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b648d347-dd3d-4511-8fd6-84bc235918eb\",\n \"name\": \"b648d347-dd3d-4511-8fd6-84bc235918eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:43:43.636760265Z\",\n \"endTime\": \"2026-09-26T23:02:47.067520117Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l4p0g2x0p8h2h6n-autoscale-noaz-ncv9k-skqk8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l4p0g2x0p8h2h6n-autoscale-noaz-ncv9k-skqk8: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001d427a0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2f88crxn77t5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b648d347-dd3d-4511-8fd6-84bc235918eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b648d347-dd3d-4511-8fd6-84bc235918eb\",\n \"name\": \"b648d347-dd3d-4511-8fd6-84bc235918eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:43:43.636760265Z\",\n \"endTime\": \"2026-09-26T23:02:47.067520117Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l4p0g2x0p8h2h6n-autoscale-noaz-ncv9k-skqk8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l4p0g2x0p8h2h6n-autoscale-noaz-ncv9k-skqk8: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103956778688450560","occurred_at":"2026-09-26T21:16:00Z","signature_id":"10d0c74d7821e472e9f3cd0ab2d72680f3767a27a31d204abf40c872c382c5df","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":21,"matched_occurrences":21,"runs_affected":2},"rows":[{"environment":"prod","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":6,"after_last_push_count":9,"trend_counts":[0,0,1,1,1,0,6],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-mstjg9\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c3ac00\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-mstjg9: failed waiting for cluster=\"cp-ystream-upgrade-4-23-mstjg9\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-rg9bswhmc52t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35af7176-419e-4509-b694-12bc3b29a042\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35af7176-419e-4509-b694-12bc3b29a042\",\n \"name\": \"35af7176-419e-4509-b694-12bc3b29a042\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:39:24.695181068Z\",\n \"endTime\": \"2026-09-27T00:08:26.202205182Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-mstjg9\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c3ac00\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-mstjg9: failed waiting for cluster=\"cp-ystream-upgrade-4-23-mstjg9\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-rg9bswhmc52t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35af7176-419e-4509-b694-12bc3b29a042\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35af7176-419e-4509-b694-12bc3b29a042\",\n \"name\": \"35af7176-419e-4509-b694-12bc3b29a042\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:39:24.695181068Z\",\n \"endTime\": \"2026-09-27T00:08:26.202205182Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-pf9654nq5xvx/cluster-nightly-4-22-tf8p6h should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cac680\u003e: \n failed to create HCP cluster cluster-nightly-4-22-tf8p6h: failed waiting for cluster=\"cluster-nightly-4-22-tf8p6h\" in resourcegroup=\"rg-nightly-4-22-pf9654nq5xvx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6147af6b-9de6-4408-a39c-0aa05634ba28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6147af6b-9de6-4408-a39c-0aa05634ba28\",\n \"name\": \"6147af6b-9de6-4408-a39c-0aa05634ba28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:33:53.02148446Z\",\n \"endTime\": \"2026-09-27T00:02:56.178071499Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-pf9654nq5xvx/cluster-nightly-4-22-tf8p6h should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cac680\u003e: \n failed to create HCP cluster cluster-nightly-4-22-tf8p6h: failed waiting for cluster=\"cluster-nightly-4-22-tf8p6h\" in resourcegroup=\"rg-nightly-4-22-pf9654nq5xvx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6147af6b-9de6-4408-a39c-0aa05634ba28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6147af6b-9de6-4408-a39c-0aa05634ba28\",\n \"name\": \"6147af6b-9de6-4408-a39c-0aa05634ba28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:33:53.02148446Z\",\n \"endTime\": \"2026-09-27T00:02:56.178071499Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-c6296f\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f10cc0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-c6296f: failed waiting for cluster=\"cp-ystream-upgrade-5-0-c6296f\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-thktzkr667l8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e13dc78-6363-44a0-a4cb-6cf687eb653c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e13dc78-6363-44a0-a4cb-6cf687eb653c\",\n \"name\": \"2e13dc78-6363-44a0-a4cb-6cf687eb653c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:39:39.055206831Z\",\n \"endTime\": \"2026-09-27T00:08:46.197970023Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-c6296f\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f10cc0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-c6296f: failed waiting for cluster=\"cp-ystream-upgrade-5-0-c6296f\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-thktzkr667l8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e13dc78-6363-44a0-a4cb-6cf687eb653c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e13dc78-6363-44a0-a4cb-6cf687eb653c\",\n \"name\": \"2e13dc78-6363-44a0-a4cb-6cf687eb653c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:39:39.055206831Z\",\n \"endTime\": \"2026-09-27T00:08:46.197970023Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"15b134d9b1c2f11953fb29588acde1168f9064424aeaeb6430679ae6824b0421","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"2a779a411bf593492105316539abd1f54e91aa141745a5334ebf3964c0c8ce1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"7882bf4b3dcb83a7eeee2c65b5637a07538c258541b84315afe9977667717afb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"a998cae1ca6926e3ecab46ec3a7d97e743d778004a1cecd37cc4592f661d0b14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"c87c45b4bc08cbfaaf0b572926aa9d17b34835b20fdb957c54f446329a117b21","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"cfcce88484594d3941b2e7b054c3d0cb474628fbd9bea67f20782799fb189503","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":5,"after_last_push_count":53,"also_in":["STG"],"trend_counts":[5,5,5,4,5,5,5],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":11,"prior_last_seen_at":"2026-09-20T23:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014a40c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-gx85bj8dbjwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014a40c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-gx85bj8dbjwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0029946c0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-hxs9z5vrpbbz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0029946c0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-hxs9z5vrpbbz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f0d9b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-cl4zndzgpjp5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f0d9b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-cl4zndzgpjp5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"07ee127f2734c717db5d7963ebab6a05f82747a01086f95b5e8522d1bea8a1d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"312d047aeaf9f27a3dfa801524792c6c23a767503bfb0fdd7e4db5a9391834c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"3c227abf2120bdc1c31c60ce1f1703129beb844d188f6c346f3c5ef45d06f46f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"a29c2cf5154fd0102d2696a881ff253f543b613779166c61baa2d90666be4f55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"b4c1f426afc58efe4dfc7270dca2f6ec6e980249e58e66d3b4ad7311b8dcf458","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":3,"after_last_push_count":27,"trend_counts":[3,1,1,2,3,2,3],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":19,"prior_last_seen_at":"2026-09-20T02:00:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138eac0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-vn4grw565jrj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f58a11a-50d1-416b-9f82-356e8273b038\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f58a11a-50d1-416b-9f82-356e8273b038\",\n \"name\": \"7f58a11a-50d1-416b-9f82-356e8273b038\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:14:10.630574624Z\",\n \"endTime\": \"2026-09-26T23:33:16.736724619Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138eac0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-vn4grw565jrj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f58a11a-50d1-416b-9f82-356e8273b038\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f58a11a-50d1-416b-9f82-356e8273b038\",\n \"name\": \"7f58a11a-50d1-416b-9f82-356e8273b038\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:14:10.630574624Z\",\n \"endTime\": \"2026-09-26T23:33:16.736724619Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010653a0\u003e: \n failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-5jg8xwpwj9d7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a82865e3-4228-4f7d-bc65-f5af89262225\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a82865e3-4228-4f7d-bc65-f5af89262225\",\n \"name\": \"a82865e3-4228-4f7d-bc65-f5af89262225\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:13:30.607288907Z\",\n \"endTime\": \"2026-09-26T23:32:36.744317191Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010653a0\u003e: \n failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-5jg8xwpwj9d7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a82865e3-4228-4f7d-bc65-f5af89262225\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a82865e3-4228-4f7d-bc65-f5af89262225\",\n \"name\": \"a82865e3-4228-4f7d-bc65-f5af89262225\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:13:30.607288907Z\",\n \"endTime\": \"2026-09-26T23:32:36.744317191Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001025420\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-nl9s5w\" in resourcegroup=\"rg-np-noedge-nl9s5w-w7vmjhnw9dp9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0af49d7-6514-44a3-b7dc-5a658544196a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0af49d7-6514-44a3-b7dc-5a658544196a\",\n \"name\": \"d0af49d7-6514-44a3-b7dc-5a658544196a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T03:19:54.570193352Z\",\n \"endTime\": \"2026-09-26T03:38:58.303040915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001025420\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-nl9s5w\" in resourcegroup=\"rg-np-noedge-nl9s5w-w7vmjhnw9dp9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0af49d7-6514-44a3-b7dc-5a658544196a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0af49d7-6514-44a3-b7dc-5a658544196a\",\n \"name\": \"d0af49d7-6514-44a3-b7dc-5a658544196a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T03:19:54.570193352Z\",\n \"endTime\": \"2026-09-26T03:38:58.303040915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"0fe18cdd8de62f844def4fa357318635d54320867a1c8633ec2fece9565ce3ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"1f3b4bd3b5a7a721eff71d604efa7685fa048deab8b8cf0fe82a80b643595100","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103665929874313216","occurred_at":"2026-09-26T02:00:16Z","signature_id":"1ed3f84c5538fe158e1ea52e2cae1b12fdc1a35c9a28671ff845649d33b5b757","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[0,0,0,0,4,2,2],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxchw7s6 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-f6qdhr6p7x7g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0008da0a0\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxchw7s6 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-f6qdhr6p7x7g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0008da0a0\u003e: \n storage account log verification failed\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxphshg8 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-lnmndxvjszw5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc000503980\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxphshg8 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-lnmndxvjszw5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc000503980\u003e: \n storage account log verification failed\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"cf4d5b3bdbd5761c2141297e35cf9f5181edb29fb6619d96f30e8fb4d817dea8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103665929874313216","occurred_at":"2026-09-26T02:00:16Z","signature_id":"6222019935691fa885c4fd69945b3adceec721f7a1da3ddc026e43005e19260d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":38,"also_in":["STG"],"trend_counts":[4,4,4,4,4,4,2],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":10,"prior_last_seen_at":"2026-09-20T23:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6a7e0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-jsblr8\" in resourcegroup=\"rg-np-version-upgrade-jsblr8-mb477bc2ck8f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-jsblr8-mb477bc2ck8f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-jsblr8/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6a7e0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-jsblr8\" in resourcegroup=\"rg-np-version-upgrade-jsblr8-mb477bc2ck8f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-jsblr8-mb477bc2ck8f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-jsblr8/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-gsmjgw\" for cluster \"cluster-nightly-4-20-gsmjgw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba8c60\u003e: \n failed to create NodePool np-gsmjgw: failed starting nodepool creation \"np-gsmjgw\" for cluster \"cluster-nightly-4-20-gsmjgw\" in resourcegroup=\"rg-nightly-4-20-cr5knbgkkqlq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-cr5knbgkkqlq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-gsmjgw/nodePools/np-gsmjgw\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-gsmjgw\" for cluster \"cluster-nightly-4-20-gsmjgw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba8c60\u003e: \n failed to create NodePool np-gsmjgw: failed starting nodepool creation \"np-gsmjgw\" for cluster \"cluster-nightly-4-20-gsmjgw\" in resourcegroup=\"rg-nightly-4-20-cr5knbgkkqlq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-cr5knbgkkqlq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-gsmjgw/nodePools/np-gsmjgw\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"e47e5f96cdb4ed158c7fbfae7715d756b161b0ff44ab47008e6af6e7d32e634e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"f793a0f121645a3b3ada14d1116f5029bd947c64fbdb8bc72281abdef651667c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":12,"also_in":["STG"],"trend_counts":[1,1,1,1,1,1,2],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":11,"prior_last_seen_at":"2026-09-20T23:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00158d338\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-09-24-033657, targetMinor=4.22.0-0.nightly-multi-2026-07-08-192436) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-07-08-192436\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00158d338\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-09-24-033657, targetMinor=4.22.0-0.nightly-multi-2026-07-08-192436) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-07-08-192436\"\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00111be18\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00111be18\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"7a7148bcb274817df0df58cc4f3c17b4d58edcbe850fd8416934c0973dfba2ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"feae159acc1681659551954ca69ac2fec4f445a05e1e2e12837c74f0d7fb1aa1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4ccd202ac53e63c2d458b9376fd889abcea732d65909fd57568f98cc9601e785","failure_pattern":"ERROR CODE: InvalidRequestContent; detail code Conflict; detail message Cannot request credential while credentials are being revoked; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:267]: failed to request new admin credentials after revocation for cluster \"admin-cred-lifecycle-7t664q\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000847980\u003e: \n 20240610 credential request failed: failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rwn8v4w9qpfg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-7t664q/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"certificateSigningRequest\"\n }\n }\n --------------------------------------------------------------------------------\n ; 2026-09-01-preview fallback failed: failed to begin admin credential request for hcpCluster=\"admin-cred-lifecycle-7t664q\" in resourcegroup=\"admin-credential-lifecycle-test-rwn8v4w9qpfg\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rwn8v4w9qpfg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-7t664q/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while credentials are being revoked\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rwn8v4w9qpfg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-7t664q\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:267]: failed to request new admin credentials after revocation for cluster \"admin-cred-lifecycle-7t664q\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000847980\u003e: \n 20240610 credential request failed: failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rwn8v4w9qpfg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-7t664q/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value\",\n \"target\": \"certificateSigningRequest\"\n }\n }\n --------------------------------------------------------------------------------\n ; 2026-09-01-preview fallback failed: failed to begin admin credential request for hcpCluster=\"admin-cred-lifecycle-7t664q\" in resourcegroup=\"admin-credential-lifecycle-test-rwn8v4w9qpfg\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rwn8v4w9qpfg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-7t664q/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while credentials are being revoked\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rwn8v4w9qpfg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-7t664q\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"d3a980efa305872f339ab522589dbd2c3e5f4a4a66db733dff8e01be50e9bcda","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":8,"failed_runs":8,"raw_occurrences":36,"matched_occurrences":36,"runs_affected":8},"rows":[{"environment":"stg","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":17,"runs_affected":8,"failed_runs":8,"run_impact_percent":100,"anchor_occurrences":17,"after_last_push_count":17,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,17],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011a380\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-kb4s8cpxl5xw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66b0c680-744e-463d-83fe-32a6b03b0059\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66b0c680-744e-463d-83fe-32a6b03b0059\",\n \"name\": \"66b0c680-744e-463d-83fe-32a6b03b0059\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:30:24.096246355Z\",\n \"endTime\": \"2026-09-26T23:49:30.686897638Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m1h7k2a6w9l9r1k-np-1-hqxhh-kj929: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-kb4s8cpxl5xw--managed/providers/Microsoft.Compute/virtualMachines/m1h7k2a6w9l9r1k-np-1-hqxhh-kj929 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m1h7k2a6w9l9r1k-np-1-hqxhh-kj929: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-kb4s8cpxl5xw--managed/providers/Microsoft.Compute/virtualMachines/m1h7k2a6w9l9r1k-np-1-hqxhh-kj929 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011a380\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-kb4s8cpxl5xw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66b0c680-744e-463d-83fe-32a6b03b0059\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66b0c680-744e-463d-83fe-32a6b03b0059\",\n \"name\": \"66b0c680-744e-463d-83fe-32a6b03b0059\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:30:24.096246355Z\",\n \"endTime\": \"2026-09-26T23:49:30.686897638Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m1h7k2a6w9l9r1k-np-1-hqxhh-kj929: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-kb4s8cpxl5xw--managed/providers/Microsoft.Compute/virtualMachines/m1h7k2a6w9l9r1k-np-1-hqxhh-kj929 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m1h7k2a6w9l9r1k-np-1-hqxhh-kj929: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-kb4s8cpxl5xw--managed/providers/Microsoft.Compute/virtualMachines/m1h7k2a6w9l9r1k-np-1-hqxhh-kj929 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001907188\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hpx4nnz688cq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5c2ff73-ec69-46ee-a311-273a07e5930d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5c2ff73-ec69-46ee-a311-273a07e5930d\",\n \"name\": \"d5c2ff73-ec69-46ee-a311-273a07e5930d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:33:38.153702381Z\",\n \"endTime\": \"2026-09-26T23:52:40.648818601Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cdrhx-29bn6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hpx4nnz688cq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cdrhx-29bn6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cdrhx-29bn6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hpx4nnz688cq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cdrhx-29bn6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n encountered errors downloading console logs:\n - VM \"cilium-cluster-cilium-np-cdrhx-29bn6\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-cdrhx-29bn6\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hpx4nnz688cq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cdrhx-29bn6/retrieveBootDiagnosticsData\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: OperationNotAllowed\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"OperationNotAllowed\",\n \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-cdrhx-29bn6' since another operation is in progress.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001907188\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hpx4nnz688cq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5c2ff73-ec69-46ee-a311-273a07e5930d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5c2ff73-ec69-46ee-a311-273a07e5930d\",\n \"name\": \"d5c2ff73-ec69-46ee-a311-273a07e5930d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:33:38.153702381Z\",\n \"endTime\": \"2026-09-26T23:52:40.648818601Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cdrhx-29bn6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hpx4nnz688cq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cdrhx-29bn6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cdrhx-29bn6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hpx4nnz688cq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cdrhx-29bn6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n encountered errors downloading console logs:\n - VM \"cilium-cluster-cilium-np-cdrhx-29bn6\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-cdrhx-29bn6\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hpx4nnz688cq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cdrhx-29bn6/retrieveBootDiagnosticsData\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: OperationNotAllowed\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"OperationNotAllowed\",\n \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-cdrhx-29bn6' since another operation is in progress.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:598]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000915360\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-94wr55\" in resourcegroup=\"rg-np-downgrade-94wr55-49w8dwsv7jfz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65b789c4-2968-40e8-b87b-ea9ebda5d678\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65b789c4-2968-40e8-b87b-ea9ebda5d678\",\n \"name\": \"65b789c4-2968-40e8-b87b-ea9ebda5d678\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:23:09.106743803Z\",\n \"endTime\": \"2026-09-26T22:42:12.893995625Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-94wr55-49w8dwsv7jfz-np-dg-94wr55--managed/providers/Microsoft.Compute/virtualMachines/s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-94wr55-49w8dwsv7jfz-np-dg-94wr55--managed/providers/Microsoft.Compute/virtualMachines/s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:598]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000915360\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-94wr55\" in resourcegroup=\"rg-np-downgrade-94wr55-49w8dwsv7jfz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65b789c4-2968-40e8-b87b-ea9ebda5d678\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65b789c4-2968-40e8-b87b-ea9ebda5d678\",\n \"name\": \"65b789c4-2968-40e8-b87b-ea9ebda5d678\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:23:09.106743803Z\",\n \"endTime\": \"2026-09-26T22:42:12.893995625Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-94wr55-49w8dwsv7jfz-np-dg-94wr55--managed/providers/Microsoft.Compute/virtualMachines/s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-94wr55-49w8dwsv7jfz-np-dg-94wr55--managed/providers/Microsoft.Compute/virtualMachines/s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"089da5e62c30a2a9e3d1a6245182e71a4642a38a698b3b82c0cd96b9913dc8e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"f9d15553519d6f900df80d9635a99e16a0d600a336f2c5c4d29422be77ce186f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103956302358122496","occurred_at":"2026-09-26T21:14:06Z","signature_id":"4e0a0620c8831c0f951a548fa2513f2279819e6296978d0d4cf8e21c727d4cec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103956302358122496","occurred_at":"2026-09-26T21:14:06Z","signature_id":"dbd0f8e48eac3e4cc622d79cb9d2ec8cbadb4a96fb50d05cb0b2554a97288e9e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103923127544188928","occurred_at":"2026-09-26T19:02:17Z","signature_id":"5c06e0d910ddc2acb4d5603c5eadfde618b3dd69052efab39a6a1f1a0a93424d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103923127544188928","occurred_at":"2026-09-26T19:02:17Z","signature_id":"a76814b56f15ee47998572c134135162901cee1455aa045f2734242c1a906a9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","signature_id":"103961703b8016ef8d2dc0e2f58b7bbb98c3574d1dc3c74e67f1d240fdf03492","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","signature_id":"28839740e2f9c23000e6b0500d8c0408bf8e17372a61541f2937024db1d36025","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","signature_id":"2c1b9ed23893075bc0ec6c4a6b1037e9cca96febebc0b50ef57090534c5cb402","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","signature_id":"65253901d9a0188b19f3893d1b92022170a5abe60d3935d41601fb4c5d5f1b5a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","signature_id":"7049cd27c77f3381a35c5d814c9d2733d694154d7c87b77c2714762c51e52aaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","signature_id":"82368bdfff8d1629c3e37cf188eb5b40952905d54948f637fde213abe6a41563","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103778668198236160","occurred_at":"2026-09-26T09:28:15Z","signature_id":"3e311527d14456ac6894554b956c239196c7332831ad87223df28d3ee5ad65c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103725120211128320","occurred_at":"2026-09-26T05:55:28Z","signature_id":"306b29e323c6a56937a91e8a9b592e4457743d2ec8d8a321540ac9c29f5c5317","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103725120211128320","occurred_at":"2026-09-26T05:55:28Z","signature_id":"cb3842f13643a3ede46cd6921339aeccd780d8ffc478c77a73c368f7b7de93b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103690040713416704","occurred_at":"2026-09-26T03:36:04Z","signature_id":"038e89abf06c9fb71bc46c4424eb7ecab72c651acda302bd05745bbec4db2f13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2103665929907867648","occurred_at":"2026-09-26T02:00:16Z","signature_id":"61e1a0c5120956316d98b8f902f363d01c50acf5c533fe1e2472e0d4bbca8113","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,6],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-tdqnbhv945ln/cluster-nightly-4-22-hwtgkf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000936960\u003e: \n failed to create HCP cluster cluster-nightly-4-22-hwtgkf: failed waiting for cluster=\"cluster-nightly-4-22-hwtgkf\" in resourcegroup=\"rg-nightly-4-22-tdqnbhv945ln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c8f5770-a2ce-4214-855c-9d70572faf14\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c8f5770-a2ce-4214-855c-9d70572faf14\",\n \"name\": \"2c8f5770-a2ce-4214-855c-9d70572faf14\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:43:51.36895754Z\",\n \"endTime\": \"2026-09-27T00:13:00.824555826Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-tdqnbhv945ln/cluster-nightly-4-22-hwtgkf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000936960\u003e: \n failed to create HCP cluster cluster-nightly-4-22-hwtgkf: failed waiting for cluster=\"cluster-nightly-4-22-hwtgkf\" in resourcegroup=\"rg-nightly-4-22-tdqnbhv945ln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c8f5770-a2ce-4214-855c-9d70572faf14\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c8f5770-a2ce-4214-855c-9d70572faf14\",\n \"name\": \"2c8f5770-a2ce-4214-855c-9d70572faf14\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:43:51.36895754Z\",\n \"endTime\": \"2026-09-27T00:13:00.824555826Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-f6zhb5 at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011b080\u003e: \n failed to create HCP cluster np-skip-minor-f6zhb5: failed waiting for cluster=\"np-skip-minor-f6zhb5\" in resourcegroup=\"rg-np-skip-minor-f6zhb5-mltd4bpn2ng8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3d18d82-d0f9-496a-97bb-ff0fdefcdec6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3d18d82-d0f9-496a-97bb-ff0fdefcdec6\",\n \"name\": \"b3d18d82-d0f9-496a-97bb-ff0fdefcdec6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-27T00:03:26.334155746Z\",\n \"endTime\": \"2026-09-27T00:32:27.905092762Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-f6zhb5 at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011b080\u003e: \n failed to create HCP cluster np-skip-minor-f6zhb5: failed waiting for cluster=\"np-skip-minor-f6zhb5\" in resourcegroup=\"rg-np-skip-minor-f6zhb5-mltd4bpn2ng8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3d18d82-d0f9-496a-97bb-ff0fdefcdec6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3d18d82-d0f9-496a-97bb-ff0fdefcdec6\",\n \"name\": \"b3d18d82-d0f9-496a-97bb-ff0fdefcdec6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-27T00:03:26.334155746Z\",\n \"endTime\": \"2026-09-27T00:32:27.905092762Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000afa620\u003e: \n failed to create HCP cluster np-dg-minor-6vv4rp: failed waiting for cluster=\"np-dg-minor-6vv4rp\" in resourcegroup=\"rg-np-dg-minor-6vv4rp-259t47rttggq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66c0cba9-056f-4636-8679-f28ce98cdca9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66c0cba9-056f-4636-8679-f28ce98cdca9\",\n \"name\": \"66c0cba9-056f-4636-8679-f28ce98cdca9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-27T00:05:12.968666607Z\",\n \"endTime\": \"2026-09-27T00:34:17.912617084Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000afa620\u003e: \n failed to create HCP cluster np-dg-minor-6vv4rp: failed waiting for cluster=\"np-dg-minor-6vv4rp\" in resourcegroup=\"rg-np-dg-minor-6vv4rp-259t47rttggq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66c0cba9-056f-4636-8679-f28ce98cdca9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66c0cba9-056f-4636-8679-f28ce98cdca9\",\n \"name\": \"66c0cba9-056f-4636-8679-f28ce98cdca9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-27T00:05:12.968666607Z\",\n \"endTime\": \"2026-09-27T00:34:17.912617084Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"11fafce13cda39efdbda8cbfbabf4f5aa9d728b01c3b8cc3fd705a8eb948f07f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"2ba467d1a14629feb97a61c1ba7852b7404d2206ddc7f3762c7e73e411f2a02b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"4540928969b272dea9930016b863cc4715cd1070f913dd0d298a5c42ca07687e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"501206e43f5c4171ac14d29667891202cc96e10a3428f9f639c55ac66c20afba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"d68ec7aad4b4702fa2feadc626a6b2e344a7de36646823c29820e973bf0cd18b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"f05f1056654f1d3969661b5157a3329323ee8a790487854e6d8e017502186852","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":5,"after_last_push_count":20,"also_in":["PROD"],"trend_counts":[0,0,0,0,5,5,5],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-16T23:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001c13890\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-xslh5ql6dz6k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001c13890\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-xslh5ql6dz6k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014f6150\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-zmbfdjmggw6v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014f6150\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-zmbfdjmggw6v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001849320\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7h9j6j7d894p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001849320\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7h9j6j7d894p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"0851a1986905a5b422808848fdfdf27cfb0c40343e7c8b366a189c08d2b9281b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"838796bcc7d4a7577ada2de39969a525ff7d7bafc0d94b7082c7538d5eda55e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"ad2672a3362fafb9d20fe759bb8cd0e50a98d6e39bb8af803f5a3851f16a2e13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"db9eec2083329b01da8d2cda46e73d2a9ef0942d7ceae885ce0bc1271c38f74e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"ed12e18092a6cb363796b8a2d7f31ebc10c5f1a6cfc32b6f300e750016e2970d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":25,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014c6168\u003e: \n resource group \"tls-endpoint-cluster-s8s7nfpcrjx5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-s8s7nfpcrjx5\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/add1f9fb-6a8e-4417-8717-b28b0f76087f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/add1f9fb-6a8e-4417-8717-b28b0f76087f\",\n \"name\": \"add1f9fb-6a8e-4417-8717-b28b0f76087f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T21:34:10.468425235Z\",\n \"endTime\": \"2026-09-26T21:58:19.111858781Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2ugn3l1bvjlm39njkp5ap0fh8ek88r still exists (deletion dispatched at 2026-09-26T21:34:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014c6168\u003e: \n resource group \"tls-endpoint-cluster-s8s7nfpcrjx5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-s8s7nfpcrjx5\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/add1f9fb-6a8e-4417-8717-b28b0f76087f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/add1f9fb-6a8e-4417-8717-b28b0f76087f\",\n \"name\": \"add1f9fb-6a8e-4417-8717-b28b0f76087f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T21:34:10.468425235Z\",\n \"endTime\": \"2026-09-26T21:58:19.111858781Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2ugn3l1bvjlm39njkp5ap0fh8ek88r still exists (deletion dispatched at 2026-09-26T21:34:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c140c0\u003e: \n resource group \"tls-endpoint-cluster-tlzh4bzgw4zh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-tlzh4bzgw4zh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3547c17b-2baf-4dfa-a1b2-187f3ed372c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3547c17b-2baf-4dfa-a1b2-187f3ed372c8\",\n \"name\": \"3547c17b-2baf-4dfa-a1b2-187f3ed372c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T10:04:17.573277188Z\",\n \"endTime\": \"2026-09-26T10:28:26.278499332Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2kdfqqgc9osqgq67u99r2mbueuta1t still exists (deletion dispatched at 2026-09-26T10:04:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c140c0\u003e: \n resource group \"tls-endpoint-cluster-tlzh4bzgw4zh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-tlzh4bzgw4zh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3547c17b-2baf-4dfa-a1b2-187f3ed372c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3547c17b-2baf-4dfa-a1b2-187f3ed372c8\",\n \"name\": \"3547c17b-2baf-4dfa-a1b2-187f3ed372c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T10:04:17.573277188Z\",\n \"endTime\": \"2026-09-26T10:28:26.278499332Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2kdfqqgc9osqgq67u99r2mbueuta1t still exists (deletion dispatched at 2026-09-26T10:04:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a00138\u003e: \n resource group \"pullsecret-test-5vdpdhbfbqdk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-5vdpdhbfbqdk\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82ed94ae-6b5c-42a4-9f90-3280fa2efff4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82ed94ae-6b5c-42a4-9f90-3280fa2efff4\",\n \"name\": \"82ed94ae-6b5c-42a4-9f90-3280fa2efff4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T10:04:15.110849507Z\",\n \"endTime\": \"2026-09-26T10:28:16.325232898Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2kdg1r6crhcm349aur60m1j1fcbuh2 still exists (deletion dispatched at 2026-09-26T10:04:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a00138\u003e: \n resource group \"pullsecret-test-5vdpdhbfbqdk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-5vdpdhbfbqdk\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82ed94ae-6b5c-42a4-9f90-3280fa2efff4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82ed94ae-6b5c-42a4-9f90-3280fa2efff4\",\n \"name\": \"82ed94ae-6b5c-42a4-9f90-3280fa2efff4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T10:04:15.110849507Z\",\n \"endTime\": \"2026-09-26T10:28:16.325232898Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2kdg1r6crhcm349aur60m1j1fcbuh2 still exists (deletion dispatched at 2026-09-26T10:04:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103956302358122496","occurred_at":"2026-09-26T21:14:06Z","signature_id":"be9b62e6f592acd1637d57bc1488a35a561ba98109cf2972d839ae6a04fc56ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103778668198236160","occurred_at":"2026-09-26T09:28:15Z","signature_id":"66487e377bb5836614b65c245bd64eda43f31dc7d151c0a386ac9d9a8cd2fbe9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103778668198236160","occurred_at":"2026-09-26T09:28:15Z","signature_id":"b827dee0a9cbc7d4f17c48bae9a3424d764e85fb3cd960112ea93b2d8250799d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":2,"after_last_push_count":13,"also_in":["PROD"],"trend_counts":[0,0,0,0,3,4,2],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-16T23:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005e2bc0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-czbz85\" in resourcegroup=\"rg-np-version-upgrade-czbz85-h7z4wz569lk2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-czbz85-h7z4wz569lk2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-czbz85/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005e2bc0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-czbz85\" in resourcegroup=\"rg-np-version-upgrade-czbz85-h7z4wz569lk2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-czbz85-h7z4wz569lk2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-czbz85/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-v9gzmf\" for cluster \"cluster-nightly-4-20-v9gzmf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016ad720\u003e: \n failed to create NodePool np-v9gzmf: failed starting nodepool creation \"np-v9gzmf\" for cluster \"cluster-nightly-4-20-v9gzmf\" in resourcegroup=\"rg-nightly-4-20-lq47jh9j7pkp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-lq47jh9j7pkp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-v9gzmf/nodePools/np-v9gzmf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-v9gzmf\" for cluster \"cluster-nightly-4-20-v9gzmf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016ad720\u003e: \n failed to create NodePool np-v9gzmf: failed starting nodepool creation \"np-v9gzmf\" for cluster \"cluster-nightly-4-20-v9gzmf\" in resourcegroup=\"rg-nightly-4-20-lq47jh9j7pkp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-lq47jh9j7pkp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-v9gzmf/nodePools/np-v9gzmf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"7893b4af94afe3c0281e1d0d5e4a28b92a42f2a0ba01657f8c46a66791bde30e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"83c3157c23d8aad0c7c51357004cced7cba545ae6f2dfec2d5c5e89d40ae1156","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":2,"after_last_push_count":5,"also_in":["PROD"],"trend_counts":[0,0,0,0,1,1,2],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-16T23:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00141cf78\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-09-24-033657, targetMinor=4.22.0-0.nightly-multi-2026-07-08-192436) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-07-08-192436\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00141cf78\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-09-24-033657, targetMinor=4.22.0-0.nightly-multi-2026-07-08-192436) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-07-08-192436\"\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0017240c0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0017240c0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"0fb83beb0f702d994cf5a20ec546309e606192d5d552ebb8972f4ca02a52a5a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"1b6a70986e38df8901401ab55acb5556c4468c54133387e22f1e45a710ed2281","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"96800d7c7de38f007fb6c1e0881939b8ef8d0c398e84b2e8ff15cc298d9bdcd5","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,3,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b56dc0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-c2cwvz\" in resourcegroup=\"rg-np-noedge-c2cwvz-qj8wgrzd8jpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d8efa06-245b-47cb-9abb-ec98064be1ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d8efa06-245b-47cb-9abb-ec98064be1ca\",\n \"name\": \"4d8efa06-245b-47cb-9abb-ec98064be1ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T03:07:12.674137276Z\",\n \"endTime\": \"2026-09-26T03:26:19.484267097Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-c2cwvz-qj8wgrzd8jpf-np-ne-c2cwvz--managed/providers/Microsoft.Compute/virtualMachines/s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-c2cwvz-qj8wgrzd8jpf-np-ne-c2cwvz--managed/providers/Microsoft.Compute/virtualMachines/s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b56dc0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-c2cwvz\" in resourcegroup=\"rg-np-noedge-c2cwvz-qj8wgrzd8jpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d8efa06-245b-47cb-9abb-ec98064be1ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d8efa06-245b-47cb-9abb-ec98064be1ca\",\n \"name\": \"4d8efa06-245b-47cb-9abb-ec98064be1ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T03:07:12.674137276Z\",\n \"endTime\": \"2026-09-26T03:26:19.484267097Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-c2cwvz-qj8wgrzd8jpf-np-ne-c2cwvz--managed/providers/Microsoft.Compute/virtualMachines/s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-c2cwvz-qj8wgrzd8jpf-np-ne-c2cwvz--managed/providers/Microsoft.Compute/virtualMachines/s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2103665929907867648","occurred_at":"2026-09-26T02:00:16Z","signature_id":"f981c1008c821f13aae9959c202473fe9605a11d0fdc588a129f3cedf979334e","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}