time=2026-03-04T00:05:23.535Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-04T00:05:27.128Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-04T00:05:28.549Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=01e008b042f3b1d9d3583a6151c00926b2d82c157dbbd0a4a90f855cb1fcf047
time=2026-03-04T00:07:29.589Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0483968/providers/Microsoft.Resources/deployments/01e008b042f3b1d9d3583a6151c00926b2d82c157dbbd0a4a90f855cb1fcf047/operationStatuses/08584290241579070224\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0483968/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0483968/providers/Microsoft.Monitor/accounts/hcps-j0483968/providers/Microsoft.Authorization/roleAssignments/2fd1f45a-bf6a-5537-8079-b32100c7367b\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260304T000543Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '6e583ac6-8df7-4077-8554-590aca4bb7a9', request correlation id '7977b606-0c23-4359-a2f3-b9961854e5c8'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0483968/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0483968/providers/Microsoft.Monitor/accounts/services-j0483968/providers/Microsoft.Authorization/roleAssignments/7c700d51-06ff-53cd-9bca-75750e36f7fc\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260304T000555Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'e7da9d64-1f4e-4665-a9dd-abd66fcb4339', request correlation id '7977b606-0c23-4359-a2f3-b9961854e5c8'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T22:43:22.218Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T22:43:24.140Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T22:43:25.336Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=b94e3dc6634aea5d22d910450ffc8aaac4c703ac5767f15be2142beb67bc9e83
time=2026-03-03T22:44:56.210Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8341632/providers/Microsoft.Resources/deployments/b94e3dc6634aea5d22d910450ffc8aaac4c703ac5767f15be2142beb67bc9e83/operationStatuses/08584290290808332756\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8341632/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8341632/providers/Microsoft.Monitor/accounts/services-j8341632/providers/Microsoft.Authorization/roleAssignments/1e51f5e3-3b1b-5e1a-810b-7839e7d0ede2\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T224347Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'b1f0c85d-dc9e-4a2e-b87d-233219182ab8', request correlation id '84d04789-5ae9-4b95-b28d-c46d7760476e'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8341632/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8341632/providers/Microsoft.Monitor/accounts/hcps-j8341632/providers/Microsoft.Authorization/roleAssignments/5a996a5c-bf08-52e4-bbce-288b41214d66\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T224356Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '3380b296-2d98-4ffb-8965-9aab0fb3cdc0', request correlation id '84d04789-5ae9-4b95-b28d-c46d7760476e'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T22:34:43.688Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T22:34:45.477Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T22:34:46.768Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8f1a43a3164993ff7a79d5fabd7f200f56738fa240fd68af115d3361b0993d9b
time=2026-03-03T22:36:17.605Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9081088/providers/Microsoft.Resources/deployments/8f1a43a3164993ff7a79d5fabd7f200f56738fa240fd68af115d3361b0993d9b/operationStatuses/08584290295995219744\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9081088/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9081088/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j9081088\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"Internal error\\\",\\r\\n                \\\"message\\\": \\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 4c16e0ed-e725-429f-b341-645816fd912b:3/3/2026 10:35:51 PM (UTC) to our forums for assistance or raise a support ticket .\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9081088/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9081088/providers/Microsoft.Monitor/accounts/services-j9081088/providers/Microsoft.Authorization/roleAssignments/f7ea505e-032c-58f8-9c01-30dbb670142e\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T223506Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '8f2eec61-4537-4c20-b9ea-ea5729355e89', request correlation id '91d1922c-efcc-4ae0-94a3-9a4e7cc2c2ba'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9081088/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9081088/providers/Microsoft.Monitor/accounts/hcps-j9081088/providers/Microsoft.Authorization/roleAssignments/5c02a015-a8eb-58c3-9f82-6b559f9190e5\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T223508Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'c38338ed-74d8-45ce-8723-6099b9532542', request correlation id '91d1922c-efcc-4ae0-94a3-9a4e7cc2c2ba'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T21:21:01.840Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T21:21:04.229Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T21:21:05.527Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e07e02643917c7bd767a0273d7fdd7b255e1eb52a06f868df54a997be71cfb70
time=2026-03-03T21:22:36.238Z level=INFO msg="Retrying step" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/e07e02643917c7bd767a0273d7fdd7b255e1eb52a06f868df54a997be71cfb70/operationStatuses/08584290340208132076\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/services-j7416192/providers/Microsoft.Authorization/roleAssignments/9175f7b5-343f-5dc0-ac51-0fc6bb9ff12d\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212120Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '2527100e-393a-4427-8595-0a8e3cce0b3c', request correlation id '9504cc96-8dad-4b88-a60f-7adfcdb5d3e6'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/hcps-j7416192/providers/Microsoft.Authorization/roleAssignments/fcbc90f9-43c6-578b-bdf9-8c870d5b0358\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212119Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'efa425fd-6da4-4b88-80f7-8127c106d790', request correlation id '9504cc96-8dad-4b88-a60f-7adfcdb5d3e6'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"
time=2026-03-03T21:24:36.246Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T21:24:37.070Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T21:24:38.123Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=70ddfe4f9da4df6335f66c29000b9a1deadaa5218866767d90b58a592c874b16
time=2026-03-03T21:25:08.480Z level=INFO msg="Retrying step" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/70ddfe4f9da4df6335f66c29000b9a1deadaa5218866767d90b58a592c874b16/operationStatuses/08584290338079955644\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/hcps-j7416192/providers/Microsoft.Authorization/roleAssignments/fcbc90f9-43c6-578b-bdf9-8c870d5b0358\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212448Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'a8c40e7b-b1ed-4585-b265-e43f25c646be', request correlation id '845e4a38-7d08-487b-a22d-802b377de267'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"
time=2026-03-03T21:27:08.480Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T21:27:09.502Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T21:27:10.606Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5159deb33520cc62995efe45b56795fcf17e8ad2d0012150b793d02a5ddf558d
time=2026-03-03T21:27:40.993Z level=INFO msg="Retrying step" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/5159deb33520cc62995efe45b56795fcf17e8ad2d0012150b793d02a5ddf558d/operationStatuses/08584290336555885853\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/hcps-j7416192/providers/Microsoft.Authorization/roleAssignments/fcbc90f9-43c6-578b-bdf9-8c870d5b0358\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212720Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '062e7270-48cd-4c68-b009-90783cc14a4a', request correlation id 'd2b40f1d-c3f6-4438-8b67-5401a0bd1298'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/services-j7416192/providers/Microsoft.Authorization/roleAssignments/9175f7b5-343f-5dc0-ac51-0fc6bb9ff12d\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212718Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'cf2d3d47-0527-40a7-81e1-a5c3f3762774', request correlation id 'd2b40f1d-c3f6-4438-8b67-5401a0bd1298'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"
time=2026-03-03T21:29:40.996Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/5159deb33520cc62995efe45b56795fcf17e8ad2d0012150b793d02a5ddf558d/operationStatuses/08584290336555885853\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/hcps-j7416192/providers/Microsoft.Authorization/roleAssignments/fcbc90f9-43c6-578b-bdf9-8c870d5b0358\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212720Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '062e7270-48cd-4c68-b009-90783cc14a4a', request correlation id 'd2b40f1d-c3f6-4438-8b67-5401a0bd1298'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/services-j7416192/providers/Microsoft.Authorization/roleAssignments/9175f7b5-343f-5dc0-ac51-0fc6bb9ff12d\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212718Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'cf2d3d47-0527-40a7-81e1-a5c3f3762774', request correlation id 'd2b40f1d-c3f6-4438-8b67-5401a0bd1298'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T20:31:16.404Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T20:31:19.304Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T20:31:20.722Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=63bd186d918b27a39ac36082a0114e90dbc0af57c42f6fd0171a5baf8e0f5774
time=2026-03-03T20:33:22.079Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5598336/providers/Microsoft.Resources/deployments/63bd186d918b27a39ac36082a0114e90dbc0af57c42f6fd0171a5baf8e0f5774/operationStatuses/08584290370055641106\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5598336/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5598336/providers/Microsoft.Monitor/accounts/hcps-j5598336/providers/Microsoft.Authorization/roleAssignments/7fff89ce-0e85-5a46-8759-4383616f3460\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T203143Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '6608d85f-6cdc-4d6c-bd77-6701dc5eb52d', request correlation id 'd8cf1b39-273a-46d9-95fb-12930873bc73'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5598336/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5598336/providers/Microsoft.Monitor/accounts/services-j5598336/providers/Microsoft.Authorization/roleAssignments/fd249894-2c98-522b-9711-098aba32ca02\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T203143Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '39584e29-af60-4c2e-b158-19dd5bca4915', request correlation id 'd8cf1b39-273a-46d9-95fb-12930873bc73'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T20:15:28.523Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T20:15:31.304Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T20:15:32.986Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=0a6cdca4eae96aa66588596dc986fda8411bd731888567aca814dfe468cc9f68
time=2026-03-03T20:17:34.609Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4915840/providers/Microsoft.Resources/deployments/0a6cdca4eae96aa66588596dc986fda8411bd731888567aca814dfe468cc9f68/operationStatuses/08584290379534499269\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4915840/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4915840/providers/Microsoft.Monitor/accounts/hcps-j4915840/providers/Microsoft.Authorization/roleAssignments/cdc93feb-4d37-53e3-abcc-55be3d65534f\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T201547Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '50208b97-e613-4892-996f-15320177a2b6', request correlation id '98812432-cc4d-48c7-b0d1-8f9aac2aba39'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T19:29:43.545Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T19:29:46.036Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T19:29:47.376Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f3cadb6faa13d93d2f684951d20d530263b365f9ad8316837a6cc0d6d8b298a3
time=2026-03-03T19:31:48.364Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9412864/providers/Microsoft.Resources/deployments/f3cadb6faa13d93d2f684951d20d530263b365f9ad8316837a6cc0d6d8b298a3/operationStatuses/08584290406989340272\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9412864/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9412864/providers/Microsoft.Monitor/accounts/hcps-j9412864/providers/Microsoft.Authorization/roleAssignments/59b9b767-f3a4-52b9-b2eb-dc7285af6e97\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T193000Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'aa35890a-a807-4e8d-b273-caecb0a3af34', request correlation id '30664faf-6aaa-48bc-b554-6adc279571cc'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T18:58:21.952Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T18:58:24.041Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T18:58:25.349Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=efd59571860fd3ac9497a72fc4471c4dca54a459d1334ed42db56bb75f791b0e
time=2026-03-03T19:00:26.200Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5338368/providers/Microsoft.Resources/deployments/efd59571860fd3ac9497a72fc4471c4dca54a459d1334ed42db56bb75f791b0e/operationStatuses/08584290425809989599\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5338368/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5338368/providers/Microsoft.Monitor/accounts/services-j5338368/providers/Microsoft.Authorization/roleAssignments/6dc600af-646c-5041-b24d-ebc38c1a94d7\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T185905Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9938e214-6587-4997-b658-49dc2ed2c6c2', request correlation id 'db872278-62fe-4f8d-aade-232016bc84e7'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T18:44:20.855Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T18:44:23.490Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T18:44:24.831Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=29595863b20b733232a37c69860eb3c803694489e6ec9a8a4a9177d87db28ee1
time=2026-03-03T18:45:55.566Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8742912/providers/Microsoft.Resources/deployments/29595863b20b733232a37c69860eb3c803694489e6ec9a8a4a9177d87db28ee1/operationStatuses/08584290434214746991\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8742912/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8742912/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j8742912\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"Internal error\\\",\\r\\n                \\\"message\\\": \\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report ec8bf3a0-2358-4484-9b1d-22d51fdc1a15:3/3/2026 6:45:30 PM (UTC) to our forums for assistance or raise a support ticket .\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T18:44:42.934Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T18:44:45.237Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T18:44:46.576Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=fc02f9b633ed78d5d4159063c0b4e8625410a37846ffe24018887df2a68cd2e5
time=2026-03-03T18:46:47.292Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7014016/providers/Microsoft.Resources/deployments/fc02f9b633ed78d5d4159063c0b4e8625410a37846ffe24018887df2a68cd2e5/operationStatuses/08584290433997349954\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7014016/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7014016/providers/Microsoft.Monitor/accounts/hcps-j7014016/providers/Microsoft.Authorization/roleAssignments/1f570be8-560e-5953-b86d-28353e7eecab\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T184527Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '1f262916-94cb-4aa3-80d5-797391d27614', request correlation id '9ffe5f6e-6832-4400-a6ac-38893fc423fb'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T18:39:31.191Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T18:39:33.833Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T18:39:35.171Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=7cb5f23ee6789452609b27e401db327a0d78ac14e0046c37c7ea97592e0dcdd5
time=2026-03-03T18:41:36.432Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8632192/providers/Microsoft.Resources/deployments/7cb5f23ee6789452609b27e401db327a0d78ac14e0046c37c7ea97592e0dcdd5/operationStatuses/08584290437111508375\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8632192/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8632192/providers/Microsoft.Monitor/accounts/services-j8632192/providers/Microsoft.Authorization/roleAssignments/53fa87ca-b14e-5846-afce-e5d1e085c856\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T183958Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '2f0d908d-1954-4fa0-8ac7-8ba47fba6b35', request correlation id 'a1749b3a-86fd-46ca-a544-f592a7d6e79f'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T18:18:18.705Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T18:18:20.518Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T18:18:21.686Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=64a2b4246871719a5dd21c46273e2cbef883359f65435e8d586e48fb6533a080
time=2026-03-03T18:20:22.673Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j3471104/providers/Microsoft.Resources/deployments/64a2b4246871719a5dd21c46273e2cbef883359f65435e8d586e48fb6533a080/operationStatuses/08584290449844845101\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3471104/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3471104/providers/Microsoft.Monitor/accounts/services-j3471104/providers/Microsoft.Authorization/roleAssignments/457c2eff-bceb-52fb-b47d-f11e0dd381ff\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T181834Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'c120828d-c118-4c7d-9051-bd9340e543c2', request correlation id '8ae03719-a555-4fc3-86bc-5ba8899b3b86'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T18:24:32.908Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T18:24:35.306Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T18:24:36.417Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e69b7f69f0a6c15e18552e3b6ae80618311980544f643f32ac90d1ba590e77da
time=2026-03-03T18:26:07.320Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9161600/providers/Microsoft.Resources/deployments/e69b7f69f0a6c15e18552e3b6ae80618311980544f643f32ac90d1ba590e77da/operationStatuses/08584290446097598179\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9161600/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9161600/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j9161600\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"Internal error\\\",\\r\\n                \\\"message\\\": \\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report a546eb2b-9245-4986-b585-a26cef25fad7:3/3/2026 6:25:44 PM (UTC) to our forums for assistance or raise a support ticket .\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9161600/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9161600/providers/Microsoft.Monitor/accounts/hcps-j9161600/providers/Microsoft.Authorization/roleAssignments/4fdfd6dd-e400-5775-a6ff-5e3947c1b047\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T182459Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'f13c9138-f83c-4d0e-924d-730676cd1827', request correlation id 'c43935a5-3026-40b8-9161-20dfd7560196'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T17:01:07.284Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T17:01:09.215Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T17:01:10.624Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9cecb097d9f924d2ffeabe37c43ecc0ea42d9b48407904871d4ca924494ea7f7
time=2026-03-03T17:03:11.784Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0754944/providers/Microsoft.Resources/deployments/9cecb097d9f924d2ffeabe37c43ecc0ea42d9b48407904871d4ca924494ea7f7/operationStatuses/08584290496157781617\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0754944/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0754944/providers/Microsoft.Monitor/accounts/services-j0754944/providers/Microsoft.Authorization/roleAssignments/c8c9d922-3ae1-5b66-990d-5c1f216992cd\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T170131Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'e58e5a43-759c-4acb-a70e-3989cd93c344', request correlation id 'b81c9b2e-6ac4-49ef-96e6-03e64583a706'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T16:31:09.363Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T16:31:11.474Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T16:31:12.607Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=46b3ee3a87d785609c120b25686463796fe503721d34b32edda849c746d12d13
time=2026-03-03T16:33:13.523Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5313280/providers/Microsoft.Resources/deployments/46b3ee3a87d785609c120b25686463796fe503721d34b32edda849c746d12d13/operationStatuses/08584290514135610543\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5313280/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5313280/providers/Microsoft.Monitor/accounts/hcps-j5313280/providers/Microsoft.Authorization/roleAssignments/f0059e6b-f028-5cd7-9b37-a778c1d299f9\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T163147Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'c5bd731c-ecfd-4068-89b7-beddb4277bb0', request correlation id 'e583ca37-6ca2-4520-bc88-773589bee7c8'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T16:11:03.728Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T16:11:05.607Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T16:11:06.938Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8ea32468eebea9aaa5b82e163d2a643cd5d4f81131257d37880093dc8c993f2d
time=2026-03-03T16:13:08.050Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9340288/providers/Microsoft.Resources/deployments/8ea32468eebea9aaa5b82e163d2a643cd5d4f81131257d37880093dc8c993f2d/operationStatuses/08584290526194167713\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9340288/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9340288/providers/Microsoft.Monitor/accounts/hcps-j9340288/providers/Microsoft.Authorization/roleAssignments/0592228e-12d3-535c-828e-9fba8d937713\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T161123Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7a52f29a-8e16-41e3-abf9-476ad3bd88c6', request correlation id '8fbcc270-16e8-4b47-b27b-eae93bf839a8'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

0.1({0x7feb6c166e60, 0xc00072dec0})
              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:80
                | 
                | By("creating HCP cluster: " + clusterName)
                > err = tc.CreateHCPClusterFromParam(ctx,
                | 	GinkgoLogr,
                | 	*resourceGroup.Name,
            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc00072dec0?})
              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465
            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()
              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901
            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 91
              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888
    ------------------------------
    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86 @ 03/03/26 18:06:58.653
  "ts"="2026-03-03 18:06:58.654155" "level"=0 "msg"="Commiting timing metadata."
  "ts"="2026-03-03 18:06:58.654256" "level"=0 "msg"="Dumping deployment operations." "deployment"="customer-infra-list-test-cluster-k6k69k-5ctt57" "resourceGroup"="cluster-listing-bm8xhc"
  "ts"="2026-03-03 18:06:59.535614" "level"=0 "msg"="Dumping deployment operations." "deployment"="mi-list-test-cluster-k6k69k-5ctt57" "resourceGroup"=""
  "ts"="2026-03-03 18:07:00.580421" "level"=0 "msg"="Dumping deployment operations." "deployment"="customer-infra-list-test-cluster-9b5xbw-2hh42m" "resourceGroup"="cluster-listing-4kc4lj"
  "ts"="2026-03-03 18:07:00.926424" "level"=0 "msg"="Dumping deployment operations." "deployment"="mi-list-test-cluster-9b5xbw-2hh42m" "resourceGroup"=""
  "ts"="2026-03-03 18:07:01.804167" "level"=0 "msg"="Wrote timing metadata" "path"="/tmp/secret/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz"
  "ts"="2026-03-03 18:07:01.804285" "level"=0 "msg"="Wrote timing metadata" "path"="/logs/artifacts/test-timing/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz"
  "ts"="2026-03-03 18:07:01.804350" "level"=0 "msg"="collecting debug info"
  "ts"="2026-03-03 18:07:01.805436" "level"=0 "msg"="collecting deployments" "resourceGroup"="aro-hcp-test-msi-containers-dev-126"
  "ts"="2026-03-03 18:07:01.805442" "level"=0 "msg"="collecting deployments" "resourceGroup"="cluster-listing-4kc4lj"
  "ts"="2026-03-03 18:07:01.805464" "level"=0 "msg"="collecting deployments" "resourceGroup"="cluster-listing-bm8xhc"
  "ts"="2026-03-03 18:07:01.805525" "level"=0 "msg"="collecting deployments" "resourceGroup"="aro-hcp-test-msi-containers-dev-32"
  "ts"="2026-03-03 18:07:02.407962" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-126" "deployment"="pooledMsiScopedAssignments"
  "ts"="2026-03-03 18:07:02.685913" "level"=0 "msg"="collecting operations" "resourceGroup"="cluster-listing-bm8xhc" "deployment"="pooledNonMsiScopedAssignments-kjvruuq77us5w"
  "ts"="2026-03-03 18:07:03.139911" "level"=0 "msg"="collecting operations" "resourceGroup"="cluster-listing-4kc4lj" "deployment"="pooledNonMsiScopedAssignments-erzs7xtv7toem"
  "ts"="2026-03-03 18:07:03.451923" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-32" "deployment"="pooledMsiScopedAssignments"
  "ts"="2026-03-03 18:07:03.863173" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-126" "deployment"="msi-deployment-126"
  "ts"="2026-03-03 18:07:04.227778" "level"=0 "msg"="collecting operations" "resourceGroup"="cluster-listing-bm8xhc" "deployment"="customer-infra-list-test-cluster-k6k69k-5ctt57"
  "ts"="2026-03-03 18:07:04.769359" "level"=0 "msg"="collecting operations" "resourceGroup"="cluster-listing-4kc4lj" "deployment"="customer-infra-list-test-cluster-9b5xbw-2hh42m"
  "ts"="2026-03-03 18:07:05.030461" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-32" "deployment"="msi-deployment-32"
  "ts"="2026-03-03 18:07:06.464726" "level"=0 "msg"="finished collecting debug info"
  "ts"="2026-03-03 18:07:06.464877" "level"=0 "msg"="deleting created resources"
  "ts"="2026-03-03 18:07:06.464997" "level"=0 "msg"="deleting all hcp clusters in resource group" "resourceGroup"="cluster-listing-4kc4lj"
  "ts"="2026-03-03 18:07:06.465083" "level"=0 "msg"="deleting all hcp clusters in resource group" "resourceGroup"="cluster-listing-bm8xhc"
  "ts"="2026-03-03 18:15:41.859301" "level"=0 "msg"="no left behind managed resource groups found" "resourceGroup"="cluster-listing-bm8xhc"
  "ts"="2026-03-03 18:15:41.859328" "level"=0 "msg"="deleting resource group" "resourceGroup"="cluster-listing-bm8xhc"
Command Error: signal: killed
Deserializaion Error: no output from command
0.1({0x7feb6c166e60, 0xc00072dec0})
              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:80
                | 
                | By("creating HCP cluster: " + clusterName)
                > err = tc.CreateHCPClusterFromParam(ctx,
                | 	GinkgoLogr,
                | 	*resourceGroup.Name,
            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc00072dec0?})
              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465
            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()
              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901
            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 91
              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888
    ------------------------------
    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86 @ 03/03/26 18:06:58.653
  "ts"="2026-03-03 18:06:58.654155" "level"=0 "msg"="Commiting timing metadata."
  "ts"="2026-03-03 18:06:58.654256" "level"=0 "msg"="Dumping deployment operations." "deployment"="customer-infra-list-test-cluster-k6k69k-5ctt57" "resourceGroup"="cluster-listing-bm8xhc"
  "ts"="2026-03-03 18:06:59.535614" "level"=0 "msg"="Dumping deployment operations." "deployment"="mi-list-test-cluster-k6k69k-5ctt57" "resourceGroup"=""
  "ts"="2026-03-03 18:07:00.580421" "level"=0 "msg"="Dumping deployment operations." "deployment"="customer-infra-list-test-cluster-9b5xbw-2hh42m" "resourceGroup"="cluster-listing-4kc4lj"
  "ts"="2026-03-03 18:07:00.926424" "level"=0 "msg"="Dumping deployment operations." "deployment"="mi-list-test-cluster-9b5xbw-2hh42m" "resourceGroup"=""
  "ts"="2026-03-03 18:07:01.804167" "level"=0 "msg"="Wrote timing metadata" "path"="/tmp/secret/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz"
  "ts"="2026-03-03 18:07:01.804285" "level"=0 "msg"="Wrote timing metadata" "path"="/logs/artifacts/test-timing/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz"
  "ts"="2026-03-03 18:07:01.804350" "level"=0 "msg"="collecting debug info"
  "ts"="2026-03-03 18:07:01.805436" "level"=0 "msg"="collecting deployments" "resourceGroup"="aro-hcp-test-msi-containers-dev-126"
  "ts"="2026-03-03 18:07:01.805442" "level"=0 "msg"="collecting deployments" "resourceGroup"="cluster-listing-4kc4lj"
  "ts"="2026-03-03 18:07:01.805464" "level"=0 "msg"="collecting deployments" "resourceGroup"="cluster-listing-bm8xhc"
  "ts"="2026-03-03 18:07:01.805525" "level"=0 "msg"="collecting deployments" "resourceGroup"="aro-hcp-test-msi-containers-dev-32"
  "ts"="2026-03-03 18:07:02.407962" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-126" "deployment"="pooledMsiScopedAssignments"
  "ts"="2026-03-03 18:07:02.685913" "level"=0 "msg"="collecting operations" "resourceGroup"="cluster-listing-bm8xhc" "deployment"="pooledNonMsiScopedAssignments-kjvruuq77us5w"
  "ts"="2026-03-03 18:07:03.139911" "level"=0 "msg"="collecting operations" "resourceGroup"="cluster-listing-4kc4lj" "deployment"="pooledNonMsiScopedAssignments-erzs7xtv7toem"
  "ts"="2026-03-03 18:07:03.451923" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-32" "deployment"="pooledMsiScopedAssignments"
  "ts"="2026-03-03 18:07:03.863173" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-126" "deployment"="msi-deployment-126"
  "ts"="2026-03-03 18:07:04.227778" "level"=0 "msg"="collecting operations" "resourceGroup"="cluster-listing-bm8xhc" "deployment"="customer-infra-list-test-cluster-k6k69k-5ctt57"
  "ts"="2026-03-03 18:07:04.769359" "level"=0 "msg"="collecting operations" "resourceGroup"="cluster-listing-4kc4lj" "deployment"="customer-infra-list-test-cluster-9b5xbw-2hh42m"
  "ts"="2026-03-03 18:07:05.030461" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-32" "deployment"="msi-deployment-32"
  "ts"="2026-03-03 18:07:06.464726" "level"=0 "msg"="finished collecting debug info"
  "ts"="2026-03-03 18:07:06.464877" "level"=0 "msg"="deleting created resources"
  "ts"="2026-03-03 18:07:06.464997" "level"=0 "msg"="deleting all hcp clusters in resource group" "resourceGroup"="cluster-listing-4kc4lj"
  "ts"="2026-03-03 18:07:06.465083" "level"=0 "msg"="deleting all hcp clusters in resource group" "resourceGroup"="cluster-listing-bm8xhc"
  "ts"="2026-03-03 18:15:41.859301" "level"=0 "msg"="no left behind managed resource groups found" "resourceGroup"="cluster-listing-bm8xhc"
  "ts"="2026-03-03 18:15:41.859328" "level"=0 "msg"="deleting resource group" "resourceGroup"="cluster-listing-bm8xhc"
Command Error: signal: killed
Deserializaion Error: no output from command

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:
    <*fmt.wrapErrors | 0xc000ef4cf0>: 
    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: failed waiting for cluster="basic-hcp-cluster" in resourcegroup="basic-cluster-qgkfmt" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-qgkfmt\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc000e74810>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt",
            },
            <*fmt.wrapErrors | 0xc000ef4cc0>{
                msg: "failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-qgkfmt\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc000e74810>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:
    <*fmt.wrapErrors | 0xc000ef4cf0>: 
    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: failed waiting for cluster="basic-hcp-cluster" in resourcegroup="basic-cluster-qgkfmt" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-qgkfmt\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc000e74810>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt",
            },
            <*fmt.wrapErrors | 0xc000ef4cc0>{
                msg: "failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-qgkfmt\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc000e74810>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:
    <*fmt.wrapErrors | 0xc00057f740>: 
    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: failed waiting for cluster="hcp-cluster-np-128" in resourcegroup="clusternp128-ljl5zz" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ljl5zz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc001048840>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz",
            },
            <*fmt.wrapErrors | 0xc00057f6b0>{
                msg: "failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ljl5zz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc001048840>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:
    <*fmt.wrapErrors | 0xc00057f740>: 
    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: failed waiting for cluster="hcp-cluster-np-128" in resourcegroup="clusternp128-ljl5zz" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ljl5zz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc001048840>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz",
            },
            <*fmt.wrapErrors | 0xc00057f6b0>{
                msg: "failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ljl5zz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc001048840>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:
    <*fmt.wrapErrors | 0xc000e73e00>: 
    failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: failed waiting for cluster="np-update-nodes-hcp-cluster" in resourcegroup="nodepool-update-nodes-5mvz9d" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-5mvz9d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc001223d20>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d",
            },
            <*fmt.wrapErrors | 0xc000e73dd0>{
                msg: "failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-5mvz9d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc001223d20>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:
    <*fmt.wrapErrors | 0xc000e73e00>: 
    failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: failed waiting for cluster="np-update-nodes-hcp-cluster" in resourcegroup="nodepool-update-nodes-5mvz9d" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-5mvz9d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc001223d20>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d",
            },
            <*fmt.wrapErrors | 0xc000e73dd0>{
                msg: "failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-5mvz9d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc001223d20>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:
    <*fmt.wrapErrors | 0xc000a27050>: 
    failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: failed waiting for cluster="sre-hcp-cluster" in resourcegroup="admin-api-breakglass-9whw8v" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9whw8v\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc000b11c60>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v",
            },
            <*fmt.wrapErrors | 0xc000a27020>{
                msg: "failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9whw8v\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc000b11c60>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:
    <*fmt.wrapErrors | 0xc000a27050>: 
    failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: failed waiting for cluster="sre-hcp-cluster" in resourcegroup="admin-api-breakglass-9whw8v" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9whw8v\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc000b11c60>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v",
            },
            <*fmt.wrapErrors | 0xc000a27020>{
                msg: "failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9whw8v\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc000b11c60>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:
    <*fmt.wrapErrors | 0xc0008c8e70>: 
    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: failed waiting for cluster="pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-zg5nhq" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-zg5nhq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc0014e6640>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq",
            },
            <*fmt.wrapErrors | 0xc0008c8e40>{
                msg: "failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-zg5nhq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc0014e6640>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:
    <*fmt.wrapErrors | 0xc0008c8e70>: 
    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: failed waiting for cluster="pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-zg5nhq" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-zg5nhq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc0014e6640>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq",
            },
            <*fmt.wrapErrors | 0xc0008c8e40>{
                msg: "failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-zg5nhq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc0014e6640>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:
    <*fmt.wrapErrors | 0xc00141e990>: 
    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: failed waiting for cluster="disabled-image-registry-hcp-cluster" in resourcegroup="disabled-image-registry-ltkkdm" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ltkkdm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc000ffe1b0>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm",
            },
            <*fmt.wrapErrors | 0xc00141e960>{
                msg: "failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ltkkdm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc000ffe1b0>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:
    <*fmt.wrapErrors | 0xc00141e990>: 
    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: failed waiting for cluster="disabled-image-registry-hcp-cluster" in resourcegroup="disabled-image-registry-ltkkdm" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ltkkdm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc000ffe1b0>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm",
            },
            <*fmt.wrapErrors | 0xc00141e960>{
                msg: "failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ltkkdm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc000ffe1b0>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:
    <*fmt.wrapErrors | 0xc001552c30>: 
    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster="arm64-vm-hcp-cluster" in resourcegroup="arm64-vm-cluster-2b5fxx" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc0015a5970>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx",
            },
            <*fmt.wrapErrors | 0xc001552c00>{
                msg: "failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc0015a5970>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:
    <*fmt.wrapErrors | 0xc001552c30>: 
    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster="arm64-vm-hcp-cluster" in resourcegroup="arm64-vm-cluster-2b5fxx" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc0015a5970>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx",
            },
            <*fmt.wrapErrors | 0xc001552c00>{
                msg: "failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc0015a5970>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:
    <*fmt.wrapErrors | 0xc001182a20>: 
    failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: failed waiting for cluster="engineering-hcp-cluster" in resourcegroup="kusto-logs-l2qgfq" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-l2qgfq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc00108cd60>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq",
            },
            <*fmt.wrapErrors | 0xc0011829f0>{
                msg: "failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-l2qgfq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc00108cd60>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:
    <*fmt.wrapErrors | 0xc001182a20>: 
    failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: failed waiting for cluster="engineering-hcp-cluster" in resourcegroup="kusto-logs-l2qgfq" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-l2qgfq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc00108cd60>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq",
            },
            <*fmt.wrapErrors | 0xc0011829f0>{
                msg: "failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-l2qgfq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc00108cd60>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:
    <*fmt.wrapErrors | 0xc000fa1740>: 
    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: failed waiting for cluster="np-autoscale-cluster" in resourcegroup="np-autoscaling-6b59qg" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-6b59qg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc0012e1480>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg",
            },
            <*fmt.wrapErrors | 0xc000fa1710>{
                msg: "failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-6b59qg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc0012e1480>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:
    <*fmt.wrapErrors | 0xc000fa1740>: 
    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: failed waiting for cluster="np-autoscale-cluster" in resourcegroup="np-autoscaling-6b59qg" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-6b59qg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc0012e1480>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg",
            },
            <*fmt.wrapErrors | 0xc000fa1710>{
                msg: "failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-6b59qg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc0012e1480>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:
    <*fmt.wrapErrors | 0xc000e22f60>: 
    failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: failed waiting for cluster="basic-cluster" in resourcegroup="rg-cluster-nsg-subnet-reuse-btq5z7" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-btq5z7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc0017f1d30>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7",
            },
            <*fmt.wrapErrors | 0xc000e22f30>{
                msg: "failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-btq5z7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc0017f1d30>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:
    <*fmt.wrapErrors | 0xc000e22f60>: 
    failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: failed waiting for cluster="basic-cluster" in resourcegroup="rg-cluster-nsg-subnet-reuse-btq5z7" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-btq5z7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc0017f1d30>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7",
            },
            <*fmt.wrapErrors | 0xc000e22f30>{
                msg: "failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-btq5z7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc0017f1d30>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred

284
          > github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7f1b192d5c60, 0xc00014cd20})
              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126
                | 
                | By("waiting for first cluster to complete creation")
                > _, err = poller1.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{
                | 	Frequency: framework.StandardPollInterval,
                | })
            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc00014cd20?})
              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465
            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()
              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901
            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 105
              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888
    ------------------------------
    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/03/26 18:06:58.671
  "ts"="2026-03-03 18:06:58.672120" "level"=0 "msg"="Commiting timing metadata."
  "ts"="2026-03-03 18:06:58.672190" "level"=0 "msg"="Dumping deployment operations." "deployment"="customer-infra-basic-hcp-cluster-xr92pd" "resourceGroup"="customer-rg-7lt4zc"
  "ts"="2026-03-03 18:06:59.238010" "level"=0 "msg"="Dumping deployment operations." "deployment"="mi-basic-hcp-cluster-xr92pd" "resourceGroup"=""
  "ts"="2026-03-03 18:07:00.144029" "level"=0 "msg"="Dumping deployment operations." "deployment"="customer-infra-basic-hcp-cluster2-5qr5cd" "resourceGroup"="customer-rg-7lt4zc"
  "ts"="2026-03-03 18:07:00.277203" "level"=0 "msg"="Dumping deployment operations." "deployment"="mi-basic-hcp-cluster2-5qr5cd" "resourceGroup"=""
  "ts"="2026-03-03 18:07:01.011697" "level"=0 "msg"="Wrote timing metadata" "path"="/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz"
  "ts"="2026-03-03 18:07:01.011815" "level"=0 "msg"="Wrote timing metadata" "path"="/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz"
  "ts"="2026-03-03 18:07:01.011908" "level"=0 "msg"="collecting debug info"
  "ts"="2026-03-03 18:07:01.012969" "level"=0 "msg"="collecting deployments" "resourceGroup"="aro-hcp-test-msi-containers-dev-208"
  "ts"="2026-03-03 18:07:01.012992" "level"=0 "msg"="collecting deployments" "resourceGroup"="aro-hcp-test-msi-containers-dev-279"
  "ts"="2026-03-03 18:07:01.013050" "level"=0 "msg"="collecting deployments" "resourceGroup"="aro-hcp-test-msi-containers-dev-73"
  "ts"="2026-03-03 18:07:01.013077" "level"=0 "msg"="collecting deployments" "resourceGroup"="customer-rg-7lt4zc"
  "ts"="2026-03-03 18:07:01.522360" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-208" "deployment"="pooledMsiScopedAssignments"
  "ts"="2026-03-03 18:07:02.020278" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-279" "deployment"="pooledMsiScopedAssignments"
  "ts"="2026-03-03 18:07:02.318172" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-73" "deployment"="pooledMsiScopedAssignments"
  "ts"="2026-03-03 18:07:02.771362" "level"=0 "msg"="collecting operations" "resourceGroup"="customer-rg-7lt4zc" "deployment"="pooledNonMsiScopedAssignments-mooiin3edme32"
  "ts"="2026-03-03 18:07:03.100387" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-208" "deployment"="msi-deployment-208"
  "ts"="2026-03-03 18:07:03.464514" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-279" "deployment"="msi-deployment-279"
  "ts"="2026-03-03 18:07:03.817218" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-73" "deployment"="msi-deployment-73"
  "ts"="2026-03-03 18:07:04.185626" "level"=0 "msg"="collecting operations" "resourceGroup"="customer-rg-7lt4zc" "deployment"="customer-infra-basic-hcp-cluster2-5qr5cd"
  "ts"="2026-03-03 18:07:05.686924" "level"=0 "msg"="collecting operations" "resourceGroup"="customer-rg-7lt4zc" "deployment"="pooledNonMsiScopedAssignments-eqbgltuyzhpg4"
  "ts"="2026-03-03 18:07:06.242343" "level"=0 "msg"="collecting operations" "resourceGroup"="customer-rg-7lt4zc" "deployment"="customer-infra-basic-hcp-cluster-xr92pd"
  "ts"="2026-03-03 18:07:06.716603" "level"=0 "msg"="finished collecting debug info"
  "ts"="2026-03-03 18:07:06.716740" "level"=0 "msg"="deleting created resources"
  "ts"="2026-03-03 18:07:06.716879" "level"=0 "msg"="deleting all hcp clusters in resource group" "resourceGroup"="customer-rg-7lt4zc"
Command Error: signal: killed
Deserializaion Error: no output from command
284
          > github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7f1b192d5c60, 0xc00014cd20})
              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126
                | 
                | By("waiting for first cluster to complete creation")
                > _, err = poller1.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{
                | 	Frequency: framework.StandardPollInterval,
                | })
            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc00014cd20?})
              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465
            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()
              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901
            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 105
              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888
    ------------------------------
    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/03/26 18:06:58.671
  "ts"="2026-03-03 18:06:58.672120" "level"=0 "msg"="Commiting timing metadata."
  "ts"="2026-03-03 18:06:58.672190" "level"=0 "msg"="Dumping deployment operations." "deployment"="customer-infra-basic-hcp-cluster-xr92pd" "resourceGroup"="customer-rg-7lt4zc"
  "ts"="2026-03-03 18:06:59.238010" "level"=0 "msg"="Dumping deployment operations." "deployment"="mi-basic-hcp-cluster-xr92pd" "resourceGroup"=""
  "ts"="2026-03-03 18:07:00.144029" "level"=0 "msg"="Dumping deployment operations." "deployment"="customer-infra-basic-hcp-cluster2-5qr5cd" "resourceGroup"="customer-rg-7lt4zc"
  "ts"="2026-03-03 18:07:00.277203" "level"=0 "msg"="Dumping deployment operations." "deployment"="mi-basic-hcp-cluster2-5qr5cd" "resourceGroup"=""
  "ts"="2026-03-03 18:07:01.011697" "level"=0 "msg"="Wrote timing metadata" "path"="/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz"
  "ts"="2026-03-03 18:07:01.011815" "level"=0 "msg"="Wrote timing metadata" "path"="/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz"
  "ts"="2026-03-03 18:07:01.011908" "level"=0 "msg"="collecting debug info"
  "ts"="2026-03-03 18:07:01.012969" "level"=0 "msg"="collecting deployments" "resourceGroup"="aro-hcp-test-msi-containers-dev-208"
  "ts"="2026-03-03 18:07:01.012992" "level"=0 "msg"="collecting deployments" "resourceGroup"="aro-hcp-test-msi-containers-dev-279"
  "ts"="2026-03-03 18:07:01.013050" "level"=0 "msg"="collecting deployments" "resourceGroup"="aro-hcp-test-msi-containers-dev-73"
  "ts"="2026-03-03 18:07:01.013077" "level"=0 "msg"="collecting deployments" "resourceGroup"="customer-rg-7lt4zc"
  "ts"="2026-03-03 18:07:01.522360" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-208" "deployment"="pooledMsiScopedAssignments"
  "ts"="2026-03-03 18:07:02.020278" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-279" "deployment"="pooledMsiScopedAssignments"
  "ts"="2026-03-03 18:07:02.318172" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-73" "deployment"="pooledMsiScopedAssignments"
  "ts"="2026-03-03 18:07:02.771362" "level"=0 "msg"="collecting operations" "resourceGroup"="customer-rg-7lt4zc" "deployment"="pooledNonMsiScopedAssignments-mooiin3edme32"
  "ts"="2026-03-03 18:07:03.100387" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-208" "deployment"="msi-deployment-208"
  "ts"="2026-03-03 18:07:03.464514" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-279" "deployment"="msi-deployment-279"
  "ts"="2026-03-03 18:07:03.817218" "level"=0 "msg"="collecting operations" "resourceGroup"="aro-hcp-test-msi-containers-dev-73" "deployment"="msi-deployment-73"
  "ts"="2026-03-03 18:07:04.185626" "level"=0 "msg"="collecting operations" "resourceGroup"="customer-rg-7lt4zc" "deployment"="customer-infra-basic-hcp-cluster2-5qr5cd"
  "ts"="2026-03-03 18:07:05.686924" "level"=0 "msg"="collecting operations" "resourceGroup"="customer-rg-7lt4zc" "deployment"="pooledNonMsiScopedAssignments-eqbgltuyzhpg4"
  "ts"="2026-03-03 18:07:06.242343" "level"=0 "msg"="collecting operations" "resourceGroup"="customer-rg-7lt4zc" "deployment"="customer-infra-basic-hcp-cluster-xr92pd"
  "ts"="2026-03-03 18:07:06.716603" "level"=0 "msg"="finished collecting debug info"
  "ts"="2026-03-03 18:07:06.716740" "level"=0 "msg"="deleting created resources"
  "ts"="2026-03-03 18:07:06.716879" "level"=0 "msg"="deleting all hcp clusters in resource group" "resourceGroup"="customer-rg-7lt4zc"
Command Error: signal: killed
Deserializaion Error: no output from command

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:
    <*fmt.wrapErrors | 0xc0015d83c0>: 
    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: failed waiting for cluster="autoscaling-hcp-cluster" in resourcegroup="autoscaling-cluster-45f8zm" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-45f8zm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc0015a6a60>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm",
            },
            <*fmt.wrapErrors | 0xc0015d8390>{
                msg: "failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-45f8zm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc0015a6a60>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:
    <*fmt.wrapErrors | 0xc0015d83c0>: 
    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: failed waiting for cluster="autoscaling-hcp-cluster" in resourcegroup="autoscaling-cluster-45f8zm" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-45f8zm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc0015a6a60>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm",
            },
            <*fmt.wrapErrors | 0xc0015d8390>{
                msg: "failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-45f8zm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc0015a6a60>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:
    <*fmt.wrapErrors | 0xc0012cac60>: 
    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: failed waiting for cluster="cidr-connectivity-test" in resourcegroup="e2e-cidr-connectivity-nt95fb" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-nt95fb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc001297c90>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb",
            },
            <*fmt.wrapErrors | 0xc0012cac30>{
                msg: "failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-nt95fb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc001297c90>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:
    <*fmt.wrapErrors | 0xc0012cac60>: 
    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: failed waiting for cluster="cidr-connectivity-test" in resourcegroup="e2e-cidr-connectivity-nt95fb" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: context deadline exceeded
    {
        msg: "failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-nt95fb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc001297c90>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb",
            },
            <*fmt.wrapErrors | 0xc0012cac30>{
                msg: "failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-nt95fb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: context deadline exceeded",
                errs: [
                    <*errors.errorString | 0xc001297c90>{
                        s: "timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb",
                    },
                    <context.deadlineExceededError>{},
                ],
            },
        ],
    }
occurred

time=2026-03-03T14:46:05.987Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T14:46:08.124Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T14:46:09.313Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d1a47a3051f2c857f26b8adf5743ab7f600aa055b7522f13b681251f97dfba4b
time=2026-03-03T14:47:40.143Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7790208/providers/Microsoft.Resources/deployments/d1a47a3051f2c857f26b8adf5743ab7f600aa055b7522f13b681251f97dfba4b/operationStatuses/08584290577168944907\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7790208/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7790208/providers/Microsoft.Monitor/accounts/services-j7790208/providers/Microsoft.Authorization/roleAssignments/95ec45db-bb4e-515b-961d-153f8b7e7e47\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T144630Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '610d3bc0-a518-456f-87aa-0c7af69f86aa', request correlation id '3cd1d246-f469-4cad-8013-bd7ad88caf91'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T14:40:13.311Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T14:40:15.781Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T14:40:17.253Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=3b23317ac96742e5ffa43a7dfc39860a846f8fffe6e216de8ee57eacc9589cd7
time=2026-03-03T14:41:48.100Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0537984/providers/Microsoft.Resources/deployments/3b23317ac96742e5ffa43a7dfc39860a846f8fffe6e216de8ee57eacc9589cd7/operationStatuses/08584290580690595756\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0537984/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0537984/providers/Microsoft.Monitor/accounts/services-j0537984/providers/Microsoft.Authorization/roleAssignments/0ffa2d13-e94b-5d19-8aee-3f3a85caa948\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T144031Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '21a4d2da-0b47-4f56-ad1f-2fdffde5755a', request correlation id 'e5b1cb59-2756-424b-85c4-3d0c9ba186c5'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-03T14:02:53.493Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T14:02:55.987Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T14:02:57.399Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=108f2eb68399ed8c24881221a7ea419480bbc39de21cb695ec1c25999c49a08b
time=2026-03-03T14:04:28.175Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5276160/providers/Microsoft.Resources/deployments/108f2eb68399ed8c24881221a7ea419480bbc39de21cb695ec1c25999c49a08b/operationStatuses/08584290603088964675\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5276160/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5276160/providers/Microsoft.Monitor/accounts/hcps-j5276160/providers/Microsoft.Authorization/roleAssignments/aa532dc4-e19b-50a3-99c9-65704d740d8e\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T140311Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '8dae1e94-3d2d-49cb-ba0f-43151072c8f5', request correlation id 'a8364498-ca2e-4dba-a4a6-d8290b38b749'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5276160/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5276160/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j5276160\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"Internal error\\\",\\r\\n                \\\"message\\\": \\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 492d1aa0-cd90-4eb7-af4f-b0c2bcbcaaa1:3/3/2026 2:04:11 PM (UTC) to our forums for assistance or raise a support ticket .\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5276160/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5276160/providers/Microsoft.Monitor/accounts/services-j5276160/providers/Microsoft.Authorization/roleAssignments/d07bec58-1bd9-5ce8-9e2e-ddad3342714f\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T140319Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '1d90f6eb-e8ff-48ef-910c-f85482c765fc', request correlation id 'a8364498-ca2e-4dba-a4a6-d8290b38b749'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod
Unexpected error:
    <*fmt.wrapError | 0xc000e283c0>: 
    failed to get service aro-hcp-exporter/aro-hcp-exporter: services "aro-hcp-exporter" not found
    {
        msg: "failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found",
        err: <*errors.StatusError | 0xc000b30960>{
            ErrStatus: {
                TypeMeta: {Kind: "", APIVersion: ""},
                ListMeta: {
                    SelfLink: "",
                    ResourceVersion: "",
                    Continue: "",
                    RemainingItemCount: nil,
                },
                Status: "Failure",
                Message: "services \"aro-hcp-exporter\" not found",
                Reason: "NotFound",
                Details: {
                    Name: "aro-hcp-exporter",
                    Group: "",
                    Kind: "services",
                    UID: "",
                    Causes: nil,
                    RetryAfterSeconds: 0,
                },
                Code: 404,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod
Unexpected error:
    <*fmt.wrapError | 0xc000e283c0>: 
    failed to get service aro-hcp-exporter/aro-hcp-exporter: services "aro-hcp-exporter" not found
    {
        msg: "failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found",
        err: <*errors.StatusError | 0xc000b30960>{
            ErrStatus: {
                TypeMeta: {Kind: "", APIVersion: ""},
                ListMeta: {
                    SelfLink: "",
                    ResourceVersion: "",
                    Continue: "",
                    RemainingItemCount: nil,
                },
                Status: "Failure",
                Message: "services \"aro-hcp-exporter\" not found",
                Reason: "NotFound",
                Details: {
                    Name: "aro-hcp-exporter",
                    Group: "",
                    Kind: "services",
                    UID: "",
                    Causes: nil,
                    RetryAfterSeconds: 0,
                },
                Code: 404,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod
Unexpected error:
    <*fmt.wrapError | 0xc00090f2a0>: 
    failed to get service aro-hcp-exporter/aro-hcp-exporter: services "aro-hcp-exporter" not found
    {
        msg: "failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found",
        err: <*errors.StatusError | 0xc000c660a0>{
            ErrStatus: {
                TypeMeta: {Kind: "", APIVersion: ""},
                ListMeta: {
                    SelfLink: "",
                    ResourceVersion: "",
                    Continue: "",
                    RemainingItemCount: nil,
                },
                Status: "Failure",
                Message: "services \"aro-hcp-exporter\" not found",
                Reason: "NotFound",
                Details: {
                    Name: "aro-hcp-exporter",
                    Group: "",
                    Kind: "services",
                    UID: "",
                    Causes: nil,
                    RetryAfterSeconds: 0,
                },
                Code: 404,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod
Unexpected error:
    <*fmt.wrapError | 0xc00090f2a0>: 
    failed to get service aro-hcp-exporter/aro-hcp-exporter: services "aro-hcp-exporter" not found
    {
        msg: "failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found",
        err: <*errors.StatusError | 0xc000c660a0>{
            ErrStatus: {
                TypeMeta: {Kind: "", APIVersion: ""},
                ListMeta: {
                    SelfLink: "",
                    ResourceVersion: "",
                    Continue: "",
                    RemainingItemCount: nil,
                },
                Status: "Failure",
                Message: "services \"aro-hcp-exporter\" not found",
                Reason: "NotFound",
                Details: {
                    Name: "aro-hcp-exporter",
                    Group: "",
                    Kind: "services",
                    UID: "",
                    Causes: nil,
                    RetryAfterSeconds: 0,
                },
                Code: 404,
            },
        },
    }
occurred

time=2026-03-03T13:07:03.306Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T13:07:05.149Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T13:07:06.504Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=594af2ff45a2dfb34b82fdf9cf8bc25823cfaf32f3cd653dd64fc00e328d64fc
time=2026-03-03T13:08:37.297Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9325952/providers/Microsoft.Resources/deployments/594af2ff45a2dfb34b82fdf9cf8bc25823cfaf32f3cd653dd64fc00e328d64fc/operationStatuses/08584290636598606279\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9325952/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9325952/providers/Microsoft.Monitor/accounts/services-j9325952/providers/Microsoft.Authorization/roleAssignments/0575d068-f21c-5b7f-8844-1eddcc2174b9\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T130727Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '0f4b6721-09c6-429b-96af-8b552889a7c5', request correlation id '203770a4-a8cc-4bb2-81d0-a53a6cde2aa7'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod
Unexpected error:
    <*fmt.wrapError | 0xc000587040>: 
    failed to get service aro-hcp-exporter/aro-hcp-exporter: services "aro-hcp-exporter" not found
    {
        msg: "failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found",
        err: <*errors.StatusError | 0xc000b62780>{
            ErrStatus: {
                TypeMeta: {Kind: "", APIVersion: ""},
                ListMeta: {
                    SelfLink: "",
                    ResourceVersion: "",
                    Continue: "",
                    RemainingItemCount: nil,
                },
                Status: "Failure",
                Message: "services \"aro-hcp-exporter\" not found",
                Reason: "NotFound",
                Details: {
                    Name: "aro-hcp-exporter",
                    Group: "",
                    Kind: "services",
                    UID: "",
                    Causes: nil,
                    RetryAfterSeconds: 0,
                },
                Code: 404,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod
Unexpected error:
    <*fmt.wrapError | 0xc000587040>: 
    failed to get service aro-hcp-exporter/aro-hcp-exporter: services "aro-hcp-exporter" not found
    {
        msg: "failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found",
        err: <*errors.StatusError | 0xc000b62780>{
            ErrStatus: {
                TypeMeta: {Kind: "", APIVersion: ""},
                ListMeta: {
                    SelfLink: "",
                    ResourceVersion: "",
                    Continue: "",
                    RemainingItemCount: nil,
                },
                Status: "Failure",
                Message: "services \"aro-hcp-exporter\" not found",
                Reason: "NotFound",
                Details: {
                    Name: "aro-hcp-exporter",
                    Group: "",
                    Kind: "services",
                    UID: "",
                    Causes: nil,
                    RetryAfterSeconds: 0,
                },
                Code: 404,
            },
        },
    }
occurred

time=2026-03-03T12:31:07.922Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T12:31:09.457Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T12:31:10.642Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cd4b7faf105951b179be587d1cc2d3817259d713c2a4ee210d4639f3dfe2cde7
time=2026-03-03T12:32:41.561Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j2890624/providers/Microsoft.Resources/deployments/cd4b7faf105951b179be587d1cc2d3817259d713c2a4ee210d4639f3dfe2cde7/operationStatuses/08584290658155093078\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2890624/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2890624/providers/Microsoft.Monitor/accounts/services-j2890624/providers/Microsoft.Authorization/roleAssignments/f674afde-a7be-58ca-a533-6629e33eed66\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T123135Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '99ff43a2-da4c-4e95-907d-ef58695df9d9', request correlation id 'af35b212-71d3-4d4e-9197-3c75f7302405'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod
Unexpected error:
    <*fmt.wrapError | 0xc000f82120>: 
    failed to get service aro-hcp-exporter/aro-hcp-exporter: services "aro-hcp-exporter" not found
    {
        msg: "failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found",
        err: <*errors.StatusError | 0xc000f94960>{
            ErrStatus: {
                TypeMeta: {Kind: "", APIVersion: ""},
                ListMeta: {
                    SelfLink: "",
                    ResourceVersion: "",
                    Continue: "",
                    RemainingItemCount: nil,
                },
                Status: "Failure",
                Message: "services \"aro-hcp-exporter\" not found",
                Reason: "NotFound",
                Details: {
                    Name: "aro-hcp-exporter",
                    Group: "",
                    Kind: "services",
                    UID: "",
                    Causes: nil,
                    RetryAfterSeconds: 0,
                },
                Code: 404,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod
Unexpected error:
    <*fmt.wrapError | 0xc000f82120>: 
    failed to get service aro-hcp-exporter/aro-hcp-exporter: services "aro-hcp-exporter" not found
    {
        msg: "failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found",
        err: <*errors.StatusError | 0xc000f94960>{
            ErrStatus: {
                TypeMeta: {Kind: "", APIVersion: ""},
                ListMeta: {
                    SelfLink: "",
                    ResourceVersion: "",
                    Continue: "",
                    RemainingItemCount: nil,
                },
                Status: "Failure",
                Message: "services \"aro-hcp-exporter\" not found",
                Reason: "NotFound",
                Details: {
                    Name: "aro-hcp-exporter",
                    Group: "",
                    Kind: "services",
                    UID: "",
                    Causes: nil,
                    RetryAfterSeconds: 0,
                },
                Code: 404,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod
Unexpected error:
    <*fmt.wrapError | 0xc000a7c8c0>: 
    failed to get service aro-hcp-exporter/aro-hcp-exporter: services "aro-hcp-exporter" not found
    {
        msg: "failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found",
        err: <*errors.StatusError | 0xc0009fa460>{
            ErrStatus: {
                TypeMeta: {Kind: "", APIVersion: ""},
                ListMeta: {
                    SelfLink: "",
                    ResourceVersion: "",
                    Continue: "",
                    RemainingItemCount: nil,
                },
                Status: "Failure",
                Message: "services \"aro-hcp-exporter\" not found",
                Reason: "NotFound",
                Details: {
                    Name: "aro-hcp-exporter",
                    Group: "",
                    Kind: "services",
                    UID: "",
                    Causes: nil,
                    RetryAfterSeconds: 0,
                },
                Code: 404,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod
Unexpected error:
    <*fmt.wrapError | 0xc000a7c8c0>: 
    failed to get service aro-hcp-exporter/aro-hcp-exporter: services "aro-hcp-exporter" not found
    {
        msg: "failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found",
        err: <*errors.StatusError | 0xc0009fa460>{
            ErrStatus: {
                TypeMeta: {Kind: "", APIVersion: ""},
                ListMeta: {
                    SelfLink: "",
                    ResourceVersion: "",
                    Continue: "",
                    RemainingItemCount: nil,
                },
                Status: "Failure",
                Message: "services \"aro-hcp-exporter\" not found",
                Reason: "NotFound",
                Details: {
                    Name: "aro-hcp-exporter",
                    Group: "",
                    Kind: "services",
                    UID: "",
                    Causes: nil,
                    RetryAfterSeconds: 0,
                },
                Code: 404,
            },
        },
    }
occurred

time=2026-03-03T11:15:46.995Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description="Step deploy\n Kind: Helm\n"
time=2026-03-03T11:15:47.729Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values="map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2378752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2378752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae databaseName:arohcpprow-rp-j2378752 databaseUrl:https://arohcpprow-rp-j2378752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-pqkfnvp1/pipeline@sha256:a5bee0558fd61a844225e269b6a1d9a4b193cc74e2d161889b49783f729e214e replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-pqkfnvp1/pipeline:pull workloadIdentityClientId:52b2eb73-d9ac-4182-b194-6b3e19839ae0 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]"
time=2026-03-03T11:15:47.729Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:15:48.047Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp
time=2026-03-03T11:15:48.459Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false
time=2026-03-03T11:15:48.459Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:15:48.629Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:20:54.787Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err="resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\ncontext deadline exceeded"
time=2026-03-03T11:20:54.787Z level=INFO msg="Finished deploying Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:20:54.787Z level=INFO msg="Running inline diagnostics." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:20:59.078Z level=INFO msg="Determined release status." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description="Release \"aro-hcp-backend-dev\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\ncontext deadline exceeded" values="map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2378752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2378752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae databaseName:arohcpprow-rp-j2378752 databaseUrl:https://arohcpprow-rp-j2378752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-pqkfnvp1/pipeline@sha256:a5bee0558fd61a844225e269b6a1d9a4b193cc74e2d161889b49783f729e214e replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-pqkfnvp1/pipeline:pull workloadIdentityClientId:52b2eb73-d9ac-4182-b194-6b3e19839ae0 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]"
time=2026-03-03T11:20:59.080Z level=INFO msg="Kube-events kusto link for troubleshooting:" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url="https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4xTwW4aMRS871c87SlI6whIU7VUOaRpT1Uq1PSUqgevPQUHr9/q2Qsi4uMrQykrBC231XjmjWefp/BIJIjciUGkO7I66aRrjyuuX2DSFxfsJCZxYVbRDvqqG/yFsETYIrHVZg8PfhREROUTZOkM7o3hLqSyorLWZoFg86cWVnPTltWO+409PrpgXZj1iMpDW4iCh0mOwwnhH5PeidqLGyRxJp4Q3RuZdt6fMGw771V9wI+FXZqzuFedbzNl78x6y/KeV3s/dT7lE4wgTYWXzkIevI6xbx53WdQCa7XUnU/n8z5ycInlROwTmgcOv9zsUbd9M/3aCZR0IbkGymwpl2rPsvMaM/z/vX1C63ndIKTLMkwByX8fITmjj4b+a+XFzw/FoqshAQnxc36wsdjQag4B5ewx6aalGmkFBLqyOiHDV+V4OH6rhjdqePN9NJqMbidv3j2XA7q+pnOc8XBy+/65HAyKDb2wC7Rwwd65ECC9mnGgQ7v6tTruU7GhVjifHi5aUct2R75sSEWeZxU1iFHP8kgWC6F63QtvEc3vAAAA///4hORaDAQAAA=="
time=2026-03-03T11:20:59.082Z level=INFO msg="Deployment complete." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:20:59.082Z level=INFO msg="Retrying step" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error="error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\ncontext deadline exceeded"
time=2026-03-03T11:21:59.083Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description="Step deploy\n Kind: Helm\n"
time=2026-03-03T11:21:59.681Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values="map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2378752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2378752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae databaseName:arohcpprow-rp-j2378752 databaseUrl:https://arohcpprow-rp-j2378752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-pqkfnvp1/pipeline@sha256:a5bee0558fd61a844225e269b6a1d9a4b193cc74e2d161889b49783f729e214e replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-pqkfnvp1/pipeline:pull workloadIdentityClientId:52b2eb73-d9ac-4182-b194-6b3e19839ae0 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]"
time=2026-03-03T11:21:59.681Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:22:00.052Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp
time=2026-03-03T11:22:00.215Z level=INFO msg="Doing dry-run of Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:22:00.216Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:22:00.377Z level=INFO msg="Found release history, upgrading..." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1
time=2026-03-03T11:22:01.435Z level=INFO msg="Removing old field manager from objects in release manifest." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:22:01.435Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="/v1, Kind=ServiceAccount" namespace=aro-hcp name=backend
time=2026-03-03T11:22:01.734Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="/v1, Kind=ConfigMap" namespace=aro-hcp name=backend-azure-runtime-config
time=2026-03-03T11:22:02.176Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="/v1, Kind=ConfigMap" namespace=aro-hcp name=backend-config
time=2026-03-03T11:22:02.341Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="rbac.authorization.k8s.io/v1, Kind=Role" namespace=aro-hcp name=leader-election
time=2026-03-03T11:22:02.999Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="rbac.authorization.k8s.io/v1, Kind=RoleBinding" namespace=aro-hcp name=backend-leader-election
time=2026-03-03T11:22:03.303Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="/v1, Kind=Service" namespace=aro-hcp name=aro-hcp-backend-metrics
time=2026-03-03T11:22:03.628Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="apps/v1, Kind=Deployment" namespace=aro-hcp name=aro-hcp-backend
time=2026-03-03T11:22:03.968Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding" namespace=aro-hcp name=backend-pull-binding
time=2026-03-03T11:22:04.324Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="security.istio.io/v1, Kind=AuthorizationPolicy" namespace=aro-hcp name=allow-metrics-backend
time=2026-03-03T11:22:04.663Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="security.istio.io/v1beta1, Kind=PeerAuthentication" namespace=aro-hcp name=aro-hcp-backend-metrics
time=2026-03-03T11:22:05.112Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass" namespace=aro-hcp name=backend-service-key-vault
time=2026-03-03T11:22:05.410Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="monitoring.coreos.com/v1, Kind=ServiceMonitor" namespace=aro-hcp name=aro-hcp-backend
time=2026-03-03T11:22:05.778Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false
time=2026-03-03T11:22:05.778Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:22:05.893Z level=INFO msg="Found release history, upgrading..." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1
time=2026-03-03T11:27:08.959Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err="resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\ncontext deadline exceeded"
time=2026-03-03T11:27:08.959Z level=INFO msg="Finished deploying Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:27:08.959Z level=INFO msg="Running inline diagnostics." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:27:12.870Z level=INFO msg="Determined release status." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description="Upgrade \"aro-hcp-backend-dev\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\ncontext deadline exceeded" values="map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2378752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2378752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae databaseName:arohcpprow-rp-j2378752 databaseUrl:https://arohcpprow-rp-j2378752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-pqkfnvp1/pipeline@sha256:a5bee0558fd61a844225e269b6a1d9a4b193cc74e2d161889b49783f729e214e replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-pqkfnvp1/pipeline:pull workloadIdentityClientId:52b2eb73-d9ac-4182-b194-6b3e19839ae0 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]"
time=2026-03-03T11:27:12.872Z level=INFO msg="Kube-events kusto link for troubleshooting:" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url="https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4xTTW8aMRC9768Y7SlI6wiI2kpUOaRpT1Uq1PaUqgev/QoOXs9q7AUR8eMrQykrtERc37z35kPzCo9EgsidGES6J6uTTrr2uOH6BSZ9dcHOYhIXFhUdoG+6wX8Ia4Q9ElttjvDoV0FEVH5nj08uWBcWZUVlrc0KwSoPbSEKHiY5DrmkhdXStGV1EP6AEaS58NpZyKPXMfYNImTtDNQKW7XWnU+DFnvOgzHchdRTX+b2KurYqkESZ+KA6DNaz9sGIQ3oBvgPRuad9wP3aDvvVX3Cz4VdWrK4V52PNWfvzHbP8p43x/ne6DsHJFsgJGf02cGv2PPfcZ44uMRy3a6PHP64xZNu+2vq106gpAvJNVBmT7lWe5GdXyzDb/xU8ftjsepqSEBC/JIfNhY72iwhoDxLTLppqUbaAIFurE7I8E05HU/fq/GdGt/9nExm0+ls/O65HNHtLV3kfJhNps/laFTs6IVdoJUL9t6FAOnFjAOd0tWP1Xmeih21wrl6GrSilu2BfJ1JRZ4XFTWIUS+yJYuFUL3tLW8Rzd8AAAD//3NXtwQMBAAA"
time=2026-03-03T11:27:12.875Z level=INFO msg="Deployment complete." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:27:12.875Z level=INFO msg="Retrying step" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error="error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\ncontext deadline exceeded"
time=2026-03-03T11:28:12.880Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description="Step deploy\n Kind: Helm\n"
time=2026-03-03T11:28:13.896Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values="map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2378752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2378752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae databaseName:arohcpprow-rp-j2378752 databaseUrl:https://arohcpprow-rp-j2378752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-pqkfnvp1/pipeline@sha256:a5bee0558fd61a844225e269b6a1d9a4b193cc74e2d161889b49783f729e214e replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-pqkfnvp1/pipeline:pull workloadIdentityClientId:52b2eb73-d9ac-4182-b194-6b3e19839ae0 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]"
time=2026-03-03T11:28:13.896Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:28:14.240Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp
time=2026-03-03T11:28:14.471Z level=INFO msg="Doing dry-run of Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:28:14.471Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:28:14.631Z level=INFO msg="Found release history, upgrading..." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2
time=2026-03-03T11:28:15.790Z level=INFO msg="Removing old field manager from objects in release manifest." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:28:15.791Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="/v1, Kind=ServiceAccount" namespace=aro-hcp name=backend
time=2026-03-03T11:28:16.106Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="/v1, Kind=ConfigMap" namespace=aro-hcp name=backend-azure-runtime-config
time=2026-03-03T11:28:16.459Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="/v1, Kind=ConfigMap" namespace=aro-hcp name=backend-config
time=2026-03-03T11:28:16.631Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="rbac.authorization.k8s.io/v1, Kind=Role" namespace=aro-hcp name=leader-election
time=2026-03-03T11:28:17.182Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="rbac.authorization.k8s.io/v1, Kind=RoleBinding" namespace=aro-hcp name=backend-leader-election
time=2026-03-03T11:28:17.524Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="/v1, Kind=Service" namespace=aro-hcp name=aro-hcp-backend-metrics
time=2026-03-03T11:28:17.898Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="apps/v1, Kind=Deployment" namespace=aro-hcp name=aro-hcp-backend
time=2026-03-03T11:28:18.302Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding" namespace=aro-hcp name=backend-pull-binding
time=2026-03-03T11:28:18.595Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="security.istio.io/v1, Kind=AuthorizationPolicy" namespace=aro-hcp name=allow-metrics-backend
time=2026-03-03T11:28:19.133Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="security.istio.io/v1beta1, Kind=PeerAuthentication" namespace=aro-hcp name=aro-hcp-backend-metrics
time=2026-03-03T11:28:19.565Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass" namespace=aro-hcp name=backend-service-key-vault
time=2026-03-03T11:28:19.982Z level=INFO msg="Decoded resource from manifests." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk="monitoring.coreos.com/v1, Kind=ServiceMonitor" namespace=aro-hcp name=aro-hcp-backend
time=2026-03-03T11:28:20.327Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false
time=2026-03-03T11:28:20.327Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:28:20.498Z level=INFO msg="Found release history, upgrading..." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2
time=2026-03-03T11:33:23.598Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err="resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\ncontext deadline exceeded"
time=2026-03-03T11:33:23.598Z level=INFO msg="Finished deploying Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:33:23.599Z level=INFO msg="Running inline diagnostics." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:33:28.035Z level=INFO msg="Determined release status." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description="Upgrade \"aro-hcp-backend-dev\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\ncontext deadline exceeded" values="map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2378752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2378752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae databaseName:arohcpprow-rp-j2378752 databaseUrl:https://arohcpprow-rp-j2378752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-pqkfnvp1/pipeline@sha256:a5bee0558fd61a844225e269b6a1d9a4b193cc74e2d161889b49783f729e214e replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-pqkfnvp1/pipeline:pull workloadIdentityClientId:52b2eb73-d9ac-4182-b194-6b3e19839ae0 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]"
time=2026-03-03T11:33:28.037Z level=INFO msg="Kube-events kusto link for troubleshooting:" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url="https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5STUWvbPBSG7/0rDr5KwCppAh8lH73Iul2NjrDtqmMXsvQuUSPrmCM5ISU/ftiZF1Ncmt2+ep5XluSTeSQSRG7EINI9WZ100qXHhMtnmPTZBbuMSVzYFHSOvugKfyPsEbok1tr08fRHRkSUr5q0ZXEvOjkOa/bOHPOCcu09H1SFJM5EVWqzQ7DdgrDamjovzvoakLYCITnTdQyg3ut7RvxvMIK0Ft47C3nwOnZUL0bI3hmoHY5qrxufRis65pGDSywj2484Dxx+uc2jroeb6ZdGoKQJyVVQpkOudd+kv7JHG3toC1HwMK9uaUh+cMG6sBk2vy9+RO35WCGk6w7/58JWxnBzdt5l/+1RV0bWjfcjh6kb71V5yXsx+/l/tmtKSEBC/NT+sDE70WELAbXPEZOuaiqRDkCgidUJbTzJ57P5f2q2ULPF99vb5fxuOZ895VO6uaG3mMViOb97yqfT7ETP7ALtXLD3LgTIYMw40GW6hmP1ep6yE9XC7erlQwuq2Z7h60oK8rwpqEKMetNWslgIlcfB4S2i+R0AAP//2E6uVAwEAAA="
time=2026-03-03T11:33:28.039Z level=INFO msg="Deployment complete." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy
time=2026-03-03T11:33:28.039Z level=INFO msg="Retrying step" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error="error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\ncontext deadline exceeded"
time=2026-03-03T11:34:28.039Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err="error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\ncontext deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod
Unexpected error:
    <*fmt.wrapError | 0xc0006ce440>: 
    failed to get service aro-hcp-exporter/aro-hcp-exporter: services "aro-hcp-exporter" not found
    {
        msg: "failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found",
        err: <*errors.StatusError | 0xc0008e2500>{
            ErrStatus: {
                TypeMeta: {Kind: "", APIVersion: ""},
                ListMeta: {
                    SelfLink: "",
                    ResourceVersion: "",
                    Continue: "",
                    RemainingItemCount: nil,
                },
                Status: "Failure",
                Message: "services \"aro-hcp-exporter\" not found",
                Reason: "NotFound",
                Details: {
                    Name: "aro-hcp-exporter",
                    Group: "",
                    Kind: "services",
                    UID: "",
                    Causes: nil,
                    RetryAfterSeconds: 0,
                },
                Code: 404,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod
Unexpected error:
    <*fmt.wrapError | 0xc0006ce440>: 
    failed to get service aro-hcp-exporter/aro-hcp-exporter: services "aro-hcp-exporter" not found
    {
        msg: "failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found",
        err: <*errors.StatusError | 0xc0008e2500>{
            ErrStatus: {
                TypeMeta: {Kind: "", APIVersion: ""},
                ListMeta: {
                    SelfLink: "",
                    ResourceVersion: "",
                    Continue: "",
                    RemainingItemCount: nil,
                },
                Status: "Failure",
                Message: "services \"aro-hcp-exporter\" not found",
                Reason: "NotFound",
                Details: {
                    Name: "aro-hcp-exporter",
                    Group: "",
                    Kind: "services",
                    UID: "",
                    Causes: nil,
                    RetryAfterSeconds: 0,
                },
                Code: 404,
            },
        },
    }
occurred

time=2026-03-03T01:16:14.111Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description="Step infra\n  Kind: ARM\n  Template: templates/region.bicep\n  Parameters: configurations/region.tmpl.bicepparam"
time=2026-03-03T01:16:16.164Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra
time=2026-03-03T01:16:18.148Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=7f34c0b37c51e1001a36fba7393eddbf215791b740b755aa3e8eafb1f5011a31
time=2026-03-03T01:18:19.057Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4277248/providers/Microsoft.Resources/deployments/7f34c0b37c51e1001a36fba7393eddbf215791b740b755aa3e8eafb1f5011a31/operationStatuses/08584291063084145352\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4277248/providers/Microsoft.Resources/deployments/svc-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4277248/providers/Microsoft.Monitor/accounts/services-j4277248/providers/Microsoft.Authorization/roleAssignments/377c405e-8948-5cfe-b6a5-d9d917defab9\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T011640Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7ecd3fb0-d9ea-4f95-b0f0-7665a6b78546', request correlation id 'f88c8346-6f16-4ebb-a7d7-9fbd36b5743a'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4277248/providers/Microsoft.Resources/deployments/hcp-monitor\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4277248/providers/Microsoft.Monitor/accounts/hcps-j4277248/providers/Microsoft.Authorization/roleAssignments/1590a3e2-478a-5448-a585-2cbc68b81369\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260303T011641Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7883a3ad-0349-40d7-b636-ad3d2dafb2d9', request correlation id 'f88c8346-6f16-4ebb-a7d7-9fbd36b5743a'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"