{"meta":{"start_date":"2026-09-26","end_date":"2026-09-26","start_at":"2026-09-26T00:00:00Z","end_at":"2026-09-27T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T10:15:03Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":76,"failed_runs":75,"raw_occurrences":102,"matched_occurrences":101,"runs_affected":74},"rows":[{"environment":"dev","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":60,"runs_affected":60,"failed_runs":60,"run_impact_percent":78.94736842105263,"anchor_occurrences":60,"after_last_push_count":47,"trend_counts":[0,0,0,0,11,43,60],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":114,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":60}],"full_error_samples":["time=2026-09-26T14:49:06.836Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:49:08.412Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:49:09.638Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=6089940dde784d763a773e4c9004101f74c31b1e2eea1a61564da81e9a0036e5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5653504%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6089940dde784d763a773e4c9004101f74c31b1e2eea1a61564da81e9a0036e5\ntime=2026-09-26T14:52:40.545Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5653504/providers/Microsoft.Resources/deployments/6089940dde784d763a773e4c9004101f74c31b1e2eea1a61564da81e9a0036e5/operationStatuses/08584111727364954260\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5653504/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5653504/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5653504\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 81aead43-b997-4ca0-92b1-5f890a98ae35:9/26/2026 2:50:21 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T14:54:40.554Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:54:41.206Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:54:42.164Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9881912e12f161c343a185ab325eab0f9959440aad9a04d75fdbf5669aaa2bef portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5653504%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9881912e12f161c343a185ab325eab0f9959440aad9a04d75fdbf5669aaa2bef\ntime=2026-09-26T15:10:45.962Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5653504/providers/Microsoft.Resources/deployments/9881912e12f161c343a185ab325eab0f9959440aad9a04d75fdbf5669aaa2bef/operationStatuses/08584111724039322810\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5653504/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5653504/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5653504\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 7053750e-7976-4092-b5c9-05af8bee4bc0:9/26/2026 3:10:24 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T15:12:45.966Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T15:12:46.574Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T15:12:47.251Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5c70e5fdb4253270876a8cf87e4ad360a842c5c82a499a0efaf8267c12be42a2 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5653504%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5c70e5fdb4253270876a8cf87e4ad360a842c5c82a499a0efaf8267c12be42a2\ntime=2026-09-26T15:19:06.585Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-26T14:49:01.802Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:49:03.910Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:49:05.531Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9ec5d968c4a2abbfa09f2d13ea926004d6d393845d4e1c03095043f158076d84 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6813568%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9ec5d968c4a2abbfa09f2d13ea926004d6d393845d4e1c03095043f158076d84\ntime=2026-09-26T14:52:06.712Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6813568/providers/Microsoft.Resources/deployments/9ec5d968c4a2abbfa09f2d13ea926004d6d393845d4e1c03095043f158076d84/operationStatuses/08584111727409288089\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6813568/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6813568/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6813568\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 3938f6fe-2589-4619-be28-fc66822767aa:9/26/2026 2:50:13 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T14:54:06.713Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:54:07.745Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:54:09.171Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8afa4a83c65198c5a6152d3ce561b308250ec90817107c38ff6b5a156799c6da portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6813568%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8afa4a83c65198c5a6152d3ce561b308250ec90817107c38ff6b5a156799c6da\ntime=2026-09-26T15:09:44.252Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6813568/providers/Microsoft.Resources/deployments/8afa4a83c65198c5a6152d3ce561b308250ec90817107c38ff6b5a156799c6da/operationStatuses/08584111724372557912\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6813568/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6813568/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6813568\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 381e25a6-705f-4967-8b86-0e72fd08d4dc:9/26/2026 3:09:17 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T15:11:44.253Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T15:11:45.072Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T15:11:46.506Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f8da50925da962c46c0feb369b1ccdd3215b84037bcc35598bc1f3f1121f0d5d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6813568%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff8da50925da962c46c0feb369b1ccdd3215b84037bcc35598bc1f3f1121f0d5d\ntime=2026-09-26T15:19:01.486Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-26T14:43:15.259Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:43:16.912Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:43:17.849Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=553cb6c6b3fded1accbd2852e1223b9f885add89fe9c1ca9b9ab9fd387804af3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1104128%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F553cb6c6b3fded1accbd2852e1223b9f885add89fe9c1ca9b9ab9fd387804af3\ntime=2026-09-26T14:47:48.727Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1104128/providers/Microsoft.Resources/deployments/553cb6c6b3fded1accbd2852e1223b9f885add89fe9c1ca9b9ab9fd387804af3/operationStatuses/08584111730881613482\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1104128/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1104128/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1104128\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 5fd50c6a-1737-4597-a037-368930da6cc9:9/26/2026 2:44:27 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T14:49:48.731Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:49:49.490Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:49:50.425Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=99ebb22d91f4358f020049c1b68a81eba2ea522f9bacb97496c9a72de75bf919 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1104128%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F99ebb22d91f4358f020049c1b68a81eba2ea522f9bacb97496c9a72de75bf919\ntime=2026-09-26T15:05:54.057Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1104128/providers/Microsoft.Resources/deployments/99ebb22d91f4358f020049c1b68a81eba2ea522f9bacb97496c9a72de75bf919/operationStatuses/08584111726955290959\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1104128/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1104128/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1104128\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 169a3df3-f8a3-4c96-9074-128137b133d8:9/26/2026 3:05:31 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T15:07:54.062Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T15:07:54.711Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T15:07:55.697Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9b984337a990465b3d1ba4b559dcc1260b91bb6b32219cd0080e1d7d34f722df portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1104128%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9b984337a990465b3d1ba4b559dcc1260b91bb6b32219cd0080e1d7d34f722df\ntime=2026-09-26T15:13:14.833Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103852034485653504","occurred_at":"2026-09-26T14:19:35Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851390936813568","occurred_at":"2026-09-26T14:17:13Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851378941104128","occurred_at":"2026-09-26T14:17:11Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851381092782080","occurred_at":"2026-09-26T14:17:11Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7188,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851372238606336","occurred_at":"2026-09-26T14:17:09Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851355876626432","occurred_at":"2026-09-26T14:17:05Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851334296932352","occurred_at":"2026-09-26T14:17:00Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851316445974528","occurred_at":"2026-09-26T14:16:56Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103848603104579584","occurred_at":"2026-09-26T14:06:07Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103848790996815872","occurred_at":"2026-09-26T13:01:11Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103848898379386880","occurred_at":"2026-09-26T13:00:47Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103790396751482880","occurred_at":"2026-09-26T10:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103789910530985984","occurred_at":"2026-09-26T10:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103786893190631424","occurred_at":"2026-09-26T10:00:44Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6846,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103785005913214976","occurred_at":"2026-09-26T09:53:26Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103775301015179264","occurred_at":"2026-09-26T09:14:52Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103774808339648512","occurred_at":"2026-09-26T09:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103764124197457920","occurred_at":"2026-09-26T08:30:26Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103760196256927744","occurred_at":"2026-09-26T08:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103759712255217664","occurred_at":"2026-09-26T08:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103745097093877760","occurred_at":"2026-09-26T07:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103744610089046016","occurred_at":"2026-09-26T07:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103733651282857984","occurred_at":"2026-09-26T06:29:22Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103730519727411200","occurred_at":"2026-09-26T06:16:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103729999365279744","occurred_at":"2026-09-26T06:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103729510443651072","occurred_at":"2026-09-26T06:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103714900751683584","occurred_at":"2026-09-26T05:14:52Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103714409951006720","occurred_at":"2026-09-26T05:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103705397104742400","occurred_at":"2026-09-26T04:37:06Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103701911558426624","occurred_at":"2026-09-26T04:23:15Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103699803396378624","occurred_at":"2026-09-26T04:14:52Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103699317209436160","occurred_at":"2026-09-26T04:12:56Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103685753023238144","occurred_at":"2026-09-26T03:19:02Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103684698914951168","occurred_at":"2026-09-26T03:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103684219090767872","occurred_at":"2026-09-26T03:12:57Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670738761289728","occurred_at":"2026-09-26T02:19:23Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670723087175680","occurred_at":"2026-09-26T02:19:19Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670718657990656","occurred_at":"2026-09-26T02:19:18Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670704342831104","occurred_at":"2026-09-26T02:19:14Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670699393552384","occurred_at":"2026-09-26T02:19:13Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7168,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670647581315072","occurred_at":"2026-09-26T02:19:01Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670621719236608","occurred_at":"2026-09-26T02:18:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103667365953671168","occurred_at":"2026-09-26T02:05:58Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103667354889097216","occurred_at":"2026-09-26T02:05:56Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103656984816390144","occurred_at":"2026-09-26T01:24:43Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103655450791645184","occurred_at":"2026-09-26T01:18:38Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103655428259844096","occurred_at":"2026-09-26T01:18:32Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7188,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103654924834312192","occurred_at":"2026-09-26T01:16:32Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103650148490940416","occurred_at":"2026-09-26T00:57:33Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6775,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641099342974976","occurred_at":"2026-09-26T00:21:36Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641079956901888","occurred_at":"2026-09-26T00:21:31Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641065637548032","occurred_at":"2026-09-26T00:21:28Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641066266693632","occurred_at":"2026-09-26T00:21:28Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7168,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641060163981312","occurred_at":"2026-09-26T00:21:27Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641026831847424","occurred_at":"2026-09-26T00:21:19Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641014462844928","occurred_at":"2026-09-26T00:21:16Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641015683387392","occurred_at":"2026-09-26T00:21:16Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7176,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103640998813896704","occurred_at":"2026-09-26T00:21:12Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103639551695130624","occurred_at":"2026-09-26T00:15:27Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103639238770692096","occurred_at":"2026-09-26T00:14:12Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7181,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":6.578947368421052,"anchor_occurrences":5,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,6,5],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":11,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:30:46Z\n Ended: 2026-09-27T00:51:51Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j9219200-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t317no0qn2vco31uu1nj119agt86hub-agg-cluster\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j9219200-mgmt-1\nNamespace: ocm-arohcpci01-2t317no0qn2vco31uu1nj119agt86hub-agg-cluster","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T20:33:18Z\n Ended: 2026-09-26T20:58:20Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j6283520-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t2tnffjjbth0k2idnuevp5detncdatr-cilium-cluster\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j6283520-mgmt-2\nNamespace: ocm-arohcpci01-2t2tnffjjbth0k2idnuevp5detncdatr-cilium-cluster","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T17:32:15Z\n Ended: 2026-09-26T17:39:15Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j0431360-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t2qrbqcgq901aevacbo6vkcduiahunj-acr-pull-create\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0431360-mgmt-2\nNamespace: ocm-arohcpci01-2t2qrbqcgq901aevacbo6vkcduiahunj-acr-pull-create"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7194/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103990456999219200","occurred_at":"2026-09-26T23:29:49Z","signature_id":"6566df96cb73a9adffc87a980a6110cb9f6cb7fac4d1aef176d37f92cb479ed6","pr_number":7194,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103929857506283520","occurred_at":"2026-09-26T19:29:01Z","signature_id":"b687d77087ab0324f2e1d0aa267ff4136bba5841aa7af068e1fef75e22687772","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881473420431360","occurred_at":"2026-09-26T16:16:46Z","signature_id":"e6f461584d644efe4659caaf9224f6ef200ce34ae3721cb24a9c26cca546b081","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881450792161280","occurred_at":"2026-09-26T16:16:40Z","signature_id":"028b8b3f149367175ed04cdc868d03ee9feeb1166c251ae199607b3d027a8634","pr_number":7188,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881428969197568","occurred_at":"2026-09-26T16:16:35Z","signature_id":"29c97a1bc4096ae773b9213705339dee20d316902decd611365f2238cc2051e0","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":6.578947368421052,"anchor_occurrences":5,"after_last_push_count":65,"trend_counts":[5,20,21,22,24,30,5],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":159,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:18:12Z\n Ended: 2026-09-27T00:27:15Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6084992-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6084992-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T17:44:26Z\n Ended: 2026-09-26T17:50:28Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6814080-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6814080-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T17:44:51Z\n Ended: 2026-09-26T17:50:50Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7663616-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7663616-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"ab36f01ba1f6dd47c0278082b2e3a50d4690b691b0c683c8e73be1e12cca3d69","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881469456814080","occurred_at":"2026-09-26T16:16:45Z","signature_id":"342e9a20297ec559151d137a61844f3c9a0a2d3d9dd6a479df3a39341749c146","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881459847663616","occurred_at":"2026-09-26T16:16:42Z","signature_id":"ebafac72ebe108727ebff5313e306f4f9049eb95d8f8e7da4ad87e9d88b60349","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881428969197568","occurred_at":"2026-09-26T16:16:35Z","signature_id":"8e031d136666491c65ac1f03120c0173819d20c1c3adaf01d791c67e1aef0c7c","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881405791473664","occurred_at":"2026-09-26T16:16:29Z","signature_id":"c4223b7b494a6dc2601c8c97e4b1d46697163b469eaf658557332ddc1cec8106","pr_number":7068,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":5.2631578947368425,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,5,4],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","occurrences":4}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:33:47Z\n Ended: 2026-09-27T00:52:51Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j9219200-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t317no0qn2vco31uu1nj119agt86hub-agg-cluster\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j9219200-mgmt-1\nNamespace: ocm-arohcpci01-2t317no0qn2vco31uu1nj119agt86hub-agg-cluster","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T20:37:19Z\n Ended: 2026-09-26T21:23:32Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j6283520-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t2tnffjjbth0k2idnuevp5detncdatr-cilium-cluster\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j6283520-mgmt-2\nNamespace: ocm-arohcpci01-2t2tnffjjbth0k2idnuevp5detncdatr-cilium-cluster","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T17:35:15Z\n Ended: 2026-09-26T18:04:16Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j0431360-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t2qrbqcgq901aevacbo6vkcduiahunj-acr-pull-create\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0431360-mgmt-2\nNamespace: ocm-arohcpci01-2t2qrbqcgq901aevacbo6vkcduiahunj-acr-pull-create"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7194/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103990456999219200","occurred_at":"2026-09-26T23:29:49Z","signature_id":"118b9a2c18bb76c55af60628010650bc62252eec1df7945fcd3c5d3154495df9","pr_number":7194,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103929857506283520","occurred_at":"2026-09-26T19:29:01Z","signature_id":"9ca4bc2deb7a2ca544935fb63e821804c0dc6f87fddd7c30fc0e84f823cb1c81","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881473420431360","occurred_at":"2026-09-26T16:16:46Z","signature_id":"7035fca2cdc8aba021f22051d3cd5b3870f8b1d6ff824a3d51afe42c3c74498a","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881428969197568","occurred_at":"2026-09-26T16:16:35Z","signature_id":"8fb687948b9e439949059b6fd34880b83b1afcd34a8ef97d1f528299dcd7501c","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":5.2631578947368425,"anchor_occurrences":4,"after_last_push_count":6,"trend_counts":[0,0,2,0,1,0,4],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":17,"prior_last_seen_at":"2026-09-19T02:41:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103929841903472640","occurred_at":"2026-09-26T19:28:58Z","signature_id":"817dbbd2f4e987cf783899d57ba9ab565857a252a91d7741c25fbfc506aff5e8","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927280047755264","occurred_at":"2026-09-26T19:18:47Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881469456814080","occurred_at":"2026-09-26T16:16:45Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":3,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,5,3],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T23:57:44Z\n Ended: 2026-09-27T00:21:42Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j6084992-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t30min1009r4686mdn5h02sl9p5nb55-r4w4c9x5f5e5h4o\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j6084992-mgmt-2\nNamespace: ocm-arohcpci01-2t30min1009r4686mdn5h02sl9p5nb55-r4w4c9x5f5e5h4o","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T22:38:39Z\n Ended: 2026-09-26T23:07:40Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j0737536-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t2viddtv02l63m6ul7m30kp2hdmu31l-agg-cluster\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0737536-mgmt-1\nNamespace: ocm-arohcpci01-2t2viddtv02l63m6ul7m30kp2hdmu31l-agg-cluster","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T20:32:56Z\n Ended: 2026-09-26T20:58:54Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j2450304-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t2tneh2r23glrlaq41sk5licb45u3pf-l2y8b5i4r2k8w3t\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2450304-mgmt-2\nNamespace: ocm-arohcpci01-2t2tneh2r23glrlaq41sk5licb45u3pf-l2y8b5i4r2k8w3t\n\nFiring 2:\n State: Resolved\n Started: 2026-09-26T20:34:58Z\n Ended: 2026-09-26T20:56:56Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j2450304-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t2tna269g6d79a0hm4q37mmv6lq3q16-private-kas\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2450304-mgmt-2\nNamespace: ocm-arohcpci01-2t2tna269g6d79a0hm4q37mmv6lq3q16-private-kas"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"3a14be93314f64bd1aaf34287e4e0adef28fca1a41e77dc2ba5a7194956ee79f","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103965959260737536","occurred_at":"2026-09-26T21:52:29Z","signature_id":"11ba818ac5dd1ff347068d274c7fa5d21dba08c20baa52b6b0f46bbac6c58411","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"5afa5c76e03a3b0c3b55adc55370d70f4ab3538b83cc562c8d049fb3ad18fe8a","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.9473684210526314,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,4,3],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:00:39Z\n Ended: 2026-09-27T00:46:41Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j6084992-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t30min1009r4686mdn5h02sl9p5nb55-r4w4c9x5f5e5h4o\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j6084992-mgmt-2\nNamespace: ocm-arohcpci01-2t30min1009r4686mdn5h02sl9p5nb55-r4w4c9x5f5e5h4o","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T22:41:41Z\n Ended: 2026-09-26T23:32:41Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j0737536-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t2viddtv02l63m6ul7m30kp2hdmu31l-agg-cluster\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0737536-mgmt-1\nNamespace: ocm-arohcpci01-2t2viddtv02l63m6ul7m30kp2hdmu31l-agg-cluster","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T20:35:56Z\n Ended: 2026-09-26T21:23:56Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j2450304-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t2tneh2r23glrlaq41sk5licb45u3pf-l2y8b5i4r2k8w3t\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2450304-mgmt-2\nNamespace: ocm-arohcpci01-2t2tneh2r23glrlaq41sk5licb45u3pf-l2y8b5i4r2k8w3t\n\nFiring 2:\n State: Resolved\n Started: 2026-09-26T20:37:57Z\n Ended: 2026-09-26T21:21:56Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j2450304-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t2tna269g6d79a0hm4q37mmv6lq3q16-private-kas\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2450304-mgmt-2\nNamespace: ocm-arohcpci01-2t2tna269g6d79a0hm4q37mmv6lq3q16-private-kas"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"878ce001cd49a7ed748478aef200a126d7f5d5a033ae0db1aa1de44d5ddcc877","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103965959260737536","occurred_at":"2026-09-26T21:52:29Z","signature_id":"1d021545a8594032e7b33f8a228af4da144b809174ff6174498d77fa92101d7c","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"983ca4f9ae3ed26808cbee2ea2a4466c45d7f3a39a19058178e4a59512e41739","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Network","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,2,2],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wx6szx\" for cluster \"cluster-candidate-4-20-wx6szx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b82600\u003e: \n failed to create NodePool np-wx6szx: failed waiting for nodepool=\"np-wx6szx\" for cluster \"cluster-candidate-4-20-wx6szx\" in resourcegroup=\"rg-candidate-4-20-r2kvmlfvs4fd\" to finish creating: GET https://rp.j0737536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8ae21a1-086f-493e-86bc-e120b31e75a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8ae21a1-086f-493e-86bc-e120b31e75a1\",\n \"name\": \"a8ae21a1-086f-493e-86bc-e120b31e75a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:38:07.749745969Z\",\n \"endTime\": \"2026-09-26T22:57:10.827139148Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-20-r2kvmlfvs4fd-candidate-wx6szx--managed/x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-r2kvmlfvs4fd-candidate-wx6szx--managed/providers/Microsoft.Network/networkInterfaces/x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wx6szx\" for cluster \"cluster-candidate-4-20-wx6szx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b82600\u003e: \n failed to create NodePool np-wx6szx: failed waiting for nodepool=\"np-wx6szx\" for cluster \"cluster-candidate-4-20-wx6szx\" in resourcegroup=\"rg-candidate-4-20-r2kvmlfvs4fd\" to finish creating: GET https://rp.j0737536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8ae21a1-086f-493e-86bc-e120b31e75a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8ae21a1-086f-493e-86bc-e120b31e75a1\",\n \"name\": \"a8ae21a1-086f-493e-86bc-e120b31e75a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:38:07.749745969Z\",\n \"endTime\": \"2026-09-26T22:57:10.827139148Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-20-r2kvmlfvs4fd-candidate-wx6szx--managed/x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-r2kvmlfvs4fd-candidate-wx6szx--managed/providers/Microsoft.Network/networkInterfaces/x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vcbnqz\" for cluster \"cluster-candidate-4-20-vcbnqz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe65e0\u003e: \n failed to create NodePool np-vcbnqz: failed waiting for nodepool=\"np-vcbnqz\" for cluster \"cluster-candidate-4-20-vcbnqz\" in resourcegroup=\"rg-candidate-4-20-frjpqt6l6dlc\" to finish creating: GET https://rp.j7663616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8abd2b48-24de-4fb1-bba7-e70679eac440\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8abd2b48-24de-4fb1-bba7-e70679eac440\",\n \"name\": \"8abd2b48-24de-4fb1-bba7-e70679eac440\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T17:19:57.699169962Z\",\n \"endTime\": \"2026-09-26T17:38:57.270667313Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i2k9x4t6q6r7f1s-np-vcbnqz-qnsq9-j9kx5: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-20-frjpqt6l6dlc-candidate-vcbnqz--managed/i2k9x4t6q6r7f1s-np-vcbnqz-qnsq9-j9kx5-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-frjpqt6l6dlc-candidate-vcbnqz--managed/providers/Microsoft.Network/networkInterfaces/i2k9x4t6q6r7f1s-np-vcbnqz-qnsq9-j9kx5-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '12eafc4e-f869-4b62-8198-816b7d6d0876' exceeded. Please try again after '10' ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vcbnqz\" for cluster \"cluster-candidate-4-20-vcbnqz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe65e0\u003e: \n failed to create NodePool np-vcbnqz: failed waiting for nodepool=\"np-vcbnqz\" for cluster \"cluster-candidate-4-20-vcbnqz\" in resourcegroup=\"rg-candidate-4-20-frjpqt6l6dlc\" to finish creating: GET https://rp.j7663616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8abd2b48-24de-4fb1-bba7-e70679eac440\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8abd2b48-24de-4fb1-bba7-e70679eac440\",\n \"name\": \"8abd2b48-24de-4fb1-bba7-e70679eac440\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T17:19:57.699169962Z\",\n \"endTime\": \"2026-09-26T17:38:57.270667313Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i2k9x4t6q6r7f1s-np-vcbnqz-qnsq9-j9kx5: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-20-frjpqt6l6dlc-candidate-vcbnqz--managed/i2k9x4t6q6r7f1s-np-vcbnqz-qnsq9-j9kx5-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-frjpqt6l6dlc-candidate-vcbnqz--managed/providers/Microsoft.Network/networkInterfaces/i2k9x4t6q6r7f1s-np-vcbnqz-qnsq9-j9kx5-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '12eafc4e-f869-4b62-8198-816b7d6d0876' exceeded. Please try again after '10' ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103965959260737536","occurred_at":"2026-09-26T21:52:29Z","signature_id":"8a579545eb8ffc7ce43e3545cb26540ccd7358fcf1514bbca20793979fbd673e","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881459847663616","occurred_at":"2026-09-26T16:16:42Z","signature_id":"043f03e78369abc3dac495551386e55ba539c998c6f45575a66419b88c93eda0","pr_number":7164,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.6315789473684212,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[1,1,3,2,4,1,2],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":20,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":2}],"full_error_samples":["alert fired 4 time(s) (1 unknown, 3 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:23:15Z\n Ended: 2026-09-27T00:29:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t30n01ijoaem1ekng72tj4pcrtjldgt-u0g5w9w4b6k9u1r\", pod=\"kube-apiserver-6996784dc-8vzfd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t30n01ijoaem1ekng72tj4pcrtjldgt-u0g5w9w4b6k9u1r/kube-apiserver-6996784dc-8vzfd has been in a non-ready state for longer than 5 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T20:54:34Z\n Ended: 2026-09-26T21:00:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6283520-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t2u1irnugb4blbq80ksmoo4fkldqnt8-oidc-wi-cluster\", pod=\"kube-controller-manager-65c84bdd67-wnnzz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t2u1irnugb4blbq80ksmoo4fkldqnt8-oidc-wi-cluster/kube-controller-manager-65c84bdd67-wnnzz has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-26T21:02:36Z\n Ended: 2026-09-26T21:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6283520-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t2tnr9frt52ianj6sih8n7ki9jvmnj1-v9n4d3d5t5m9z0v\", pod=\"kube-apiserver-797f569f98-n962n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t2tnr9frt52ianj6sih8n7ki9jvmnj1-v9n4d3d5t5m9z0v/kube-apiserver-797f569f98-n962n has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"f6e1a0738cb75e810c1d760e4593e47ecd222a26626a4f413d3bafc5f44c9529","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103929857506283520","occurred_at":"2026-09-26T19:29:01Z","signature_id":"a5c5601829d7dc2e22894167e287975c02c8a19694717b357e8b26e5dffcb3d2","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,6,2],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-9cp272\" for cluster \"cluster-candidate-4-23-9cp272\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019bf20\u003e: \n failed to create NodePool np-9cp272: failed waiting for nodepool=\"np-9cp272\" for cluster \"cluster-candidate-4-23-9cp272\" in resourcegroup=\"rg-candidate-4-23-t5nrns5dgzjz\" to finish creating: GET https://rp.j6084992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92540fc4-c95a-4a5d-a847-5375c72243a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92540fc4-c95a-4a5d-a847-5375c72243a8\",\n \"name\": \"92540fc4-c95a-4a5d-a847-5375c72243a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-27T00:01:58.819547057Z\",\n \"endTime\": \"2026-09-27T00:21:07.730201043Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t5nrns5dgzjz-candidate-9cp272--managed/providers/Microsoft.Compute/virtualMachines/l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t5nrns5dgzjz-candidate-9cp272--managed/providers/Microsoft.Compute/virtualMachines/l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-9cp272\" for cluster \"cluster-candidate-4-23-9cp272\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019bf20\u003e: \n failed to create NodePool np-9cp272: failed waiting for nodepool=\"np-9cp272\" for cluster \"cluster-candidate-4-23-9cp272\" in resourcegroup=\"rg-candidate-4-23-t5nrns5dgzjz\" to finish creating: GET https://rp.j6084992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92540fc4-c95a-4a5d-a847-5375c72243a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92540fc4-c95a-4a5d-a847-5375c72243a8\",\n \"name\": \"92540fc4-c95a-4a5d-a847-5375c72243a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-27T00:01:58.819547057Z\",\n \"endTime\": \"2026-09-27T00:21:07.730201043Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t5nrns5dgzjz-candidate-9cp272--managed/providers/Microsoft.Compute/virtualMachines/l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t5nrns5dgzjz-candidate-9cp272--managed/providers/Microsoft.Compute/virtualMachines/l4k3w0h4y1l5l6t-np-9cp272-gmmfp-89h4j to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d2f360\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-9sjsgl\" in resourcegroup=\"rg-np-version-upgrade-9sjsgl-z2gscsmxlxzq\" to finish creating: GET https://rp.j6084992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ef95de3-9fbf-4193-8079-b1158861e584\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ef95de3-9fbf-4193-8079-b1158861e584\",\n \"name\": \"2ef95de3-9fbf-4193-8079-b1158861e584\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:57:59.240971764Z\",\n \"endTime\": \"2026-09-27T00:17:06.977144117Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9sjsgl-z2gscsmxlxzq-np-upgrade-9s-96d5bba1/providers/Microsoft.Compute/virtualMachines/w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9sjsgl-z2gscsmxlxzq-np-upgrade-9s-96d5bba1/providers/Microsoft.Compute/virtualMachines/w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d2f360\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-9sjsgl\" in resourcegroup=\"rg-np-version-upgrade-9sjsgl-z2gscsmxlxzq\" to finish creating: GET https://rp.j6084992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ef95de3-9fbf-4193-8079-b1158861e584\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ef95de3-9fbf-4193-8079-b1158861e584\",\n \"name\": \"2ef95de3-9fbf-4193-8079-b1158861e584\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:57:59.240971764Z\",\n \"endTime\": \"2026-09-27T00:17:06.977144117Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9sjsgl-z2gscsmxlxzq-np-upgrade-9s-96d5bba1/providers/Microsoft.Compute/virtualMachines/w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9sjsgl-z2gscsmxlxzq-np-upgrade-9s-96d5bba1/providers/Microsoft.Compute/virtualMachines/w5t3u0s9p5j2g6u-npupgrade-4-21-hgqjs-8fnck to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"3b383ee2285332a194dbad5c697b0305b25dd8206e97711a9b282832753b74b4","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"d103024f5017fdcabaff08262102c5247078ca8f9b984031104c706e84e3556e","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f78894a0f745284e7b454f80e662337a874e257b6590d707b4a761a42e23b9f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b86200\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9464kjvgqp5t\" to finish creating: GET https://rp.j0737536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\",\n \"name\": \"34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:26:42.363515696Z\",\n \"endTime\": \"2026-09-26T22:55:52.65701083Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, redhat-operators-catalog, certified-operators-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b86200\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9464kjvgqp5t\" to finish creating: GET https://rp.j0737536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\",\n \"name\": \"34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:26:42.363515696Z\",\n \"endTime\": \"2026-09-26T22:55:52.65701083Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, redhat-operators-catalog, certified-operators-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103965959260737536","occurred_at":"2026-09-26T21:52:29Z","signature_id":"214f1ab4117af1368e93bfb8d87ddb3947a25a6ea700ad87128366123ae0719e","pr_number":7185,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,2,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102e380\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-2x6vlrpgq2bh\" to finish creating: GET https://rp.j6084992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47c0d2f0-4e66-4d5c-8e28-e145397d1b94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47c0d2f0-4e66-4d5c-8e28-e145397d1b94\",\n \"name\": \"47c0d2f0-4e66-4d5c-8e28-e145397d1b94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:46:18.825261842Z\",\n \"endTime\": \"2026-09-27T00:15:27.197366694Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102e380\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-2x6vlrpgq2bh\" to finish creating: GET https://rp.j6084992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47c0d2f0-4e66-4d5c-8e28-e145397d1b94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47c0d2f0-4e66-4d5c-8e28-e145397d1b94\",\n \"name\": \"47c0d2f0-4e66-4d5c-8e28-e145397d1b94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:46:18.825261842Z\",\n \"endTime\": \"2026-09-27T00:15:27.197366694Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"9d0706e3969af7c111fcca838b1e51305acfc48ec2465de31927959306c28e66","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-18T15:17:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0020ae090\u003e: \n resource group \"fips-enabled-k6r2vqf8b9xq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-k6r2vqf8b9xq\" to finish deleting: GET https://rp.j2450304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50efbb08-7b80-41ad-a7a9-a51f5231b37f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50efbb08-7b80-41ad-a7a9-a51f5231b37f\",\n \"name\": \"50efbb08-7b80-41ad-a7a9-a51f5231b37f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T20:41:31.45560964Z\",\n \"endTime\": \"2026-09-26T21:05:35.034630666Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2tmvnf0viulta1u93uf2mlthhfuvsu still exists (deletion dispatched at 2026-09-26T20:41:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0020ae090\u003e: \n resource group \"fips-enabled-k6r2vqf8b9xq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-k6r2vqf8b9xq\" to finish deleting: GET https://rp.j2450304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50efbb08-7b80-41ad-a7a9-a51f5231b37f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50efbb08-7b80-41ad-a7a9-a51f5231b37f\",\n \"name\": \"50efbb08-7b80-41ad-a7a9-a51f5231b37f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T20:41:31.45560964Z\",\n \"endTime\": \"2026-09-26T21:05:35.034630666Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2tmvnf0viulta1u93uf2mlthhfuvsu still exists (deletion dispatched at 2026-09-26T20:41:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"aa958e32e1c37b11d08d6ef5dd635ad8ce90eced3b3170eac46f774a812933cc","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-qqprr26gtjgf\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qqprr26gtjgf, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":12,"prior_last_seen_at":"2026-09-18T18:24:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:209]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002608b40\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-qqprr26gtjgf\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qqprr26gtjgf, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:209]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002608b40\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-qqprr26gtjgf\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qqprr26gtjgf, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927280047755264","occurred_at":"2026-09-26T19:18:47Z","signature_id":"e3b07109b49bcafaeae2aeacce0e01aa2904db04b4015d601de9455d55fb2226","pr_number":7164,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":13,"trend_counts":[3,2,3,0,0,2,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":28,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-26T21:24:27Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2450304-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2450304/providers/microsoft.monitor/accounts/hcps-j2450304\", namespace=\"ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-q9544\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-26T21:24:27Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2450304-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2450304/providers/microsoft.monitor/accounts/hcps-j2450304\", namespace=\"ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-q9544\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-26T21:27:26Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2450304-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2450304/providers/microsoft.monitor/accounts/hcps-j2450304\", namespace=\"ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-q9544\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-26T21:27:27Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2450304-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2450304/providers/microsoft.monitor/accounts/hcps-j2450304\", namespace=\"ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-q9544\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t2tmvnf0viulta1u93uf2mlthhfuvsu-z7r2s8c1m6p3a8w/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"59859e98e366b5af518d04f59b67247bf6559ad517e1190c64eba0f09c96ceae","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":10,"trend_counts":[1,2,2,1,0,1,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":15,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:09:50Z\n Ended: 2026-09-27T00:37:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/hcps-j6084992\", namespace=\"ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"bb068423-1854-49c8-a132-0d10a6392326\"\n Description: Pod ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-27T00:09:50Z\n Ended: 2026-09-27T00:37:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/hcps-j6084992\", namespace=\"ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"bb068423-1854-49c8-a132-0d10a6392326\"\n Description: Pod ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"0e8d71bcb4b8b8f14ef36438303ce153649aa4d8d3bd9d130dfcadf1c764fbd1","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":51,"trend_counts":[4,2,2,2,4,6,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":101,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-26T17:41:16Z\n Ended: 2026-09-26T17:47:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9197568-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t2qtvdnlaalvsj2ub667na9fqboc359-ephemeral-disk\", pod=\"ignition-server-7fd4ccdc4c-psn2m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t2qtvdnlaalvsj2ub667na9fqboc359-ephemeral-disk/ignition-server-7fd4ccdc4c-psn2m has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881428969197568","occurred_at":"2026-09-26T16:16:35Z","signature_id":"4887464f22fd02cf878c19ad54c98e5ffd7550a5bce16e494d595bda58002c06","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":17,"trend_counts":[4,4,3,2,1,3,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":29,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:06:56Z\n Ended: 2026-09-27T00:37:55Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/hcps-j6084992\", namespace=\"ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z7blf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-27T00:06:56Z\n Ended: 2026-09-27T00:36:52Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/hcps-j6084992\", namespace=\"ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z7blf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"0dfb7950c4b9435e9a2c892dc201219aa13f6c87439b016889682cade7a522c2","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":10,"trend_counts":[2,3,2,2,1,1,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":17,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-27T00:21:19Z\n Ended: 2026-09-27T00:27:16Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/services-j6084992\", namespace=\"klusterlet-2t30mivbmcgkot8vhp8oa13ibv1vnaut\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z7blf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t30mivbmcgkot8vhp8oa13ibv1vnaut/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"a2eb7618d1e7a8cd04863ecb99aab31fa66480f2629953e2c4928c2aba4d4ae5","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-19T20:01:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881405791473664","occurred_at":"2026-09-26T16:16:29Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":7068,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f76baa3d4767ea148b5500ace79ced7c3f0e0092f22b03bde161fd5f0c1796ae","failure_pattern":"run command \"e2e-runcommand-1790458991773693237\" failed with exit code 28 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1790458991773693237\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.3157894736842106,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-20..2026-09-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 608.861s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n \u003c*errors.errorString | 0xc00107dc50\u003e: \n run command \"e2e-runcommand-1790458991773693237\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 608.861s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n \u003c*errors.errorString | 0xc00107dc50\u003e: \n run command \"e2e-runcommand-1790458991773693237\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103942172993654784","occurred_at":"2026-09-26T20:17:58Z","signature_id":"4eac6d1d6c768c796dcee39de9244b0faf13f00b731ed7de804c181901f6e831","pr_number":7068,"after_last_push_of_merged_pr":false}]}]}]}