{"meta":{"date":"2026-03-03","timezone":"UTC","generated_at":"2026-11-01T00:08:40Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":68,"failed_runs":42,"runs_with_occurrences":42,"runs_with_failure_pattern_matches":30,"runs_with_unmatched_occurrences":12,"runs_without_occurrence_rows":26,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4299/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028982772701335552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4299,"pr_state":"closed","pr_sha":"ee05cf1e3593dbabb8c2b6b4c5bc045eda2c70ab","final_merged_sha":"89abb52bd0ca3634c8e64a287368b6c007666f1e","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-03T23:56:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4287/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028975845510483968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4287,"pr_state":"closed","pr_sha":"723b6e68710379451b6a670f5df9529985329888","final_merged_sha":"723b6e68710379451b6a670f5df9529985329888","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T23:28:32Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8b47eeca7705c76cef7153cc5e3de90900f8f9eed25b67d5a218140ac862be98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4287/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028975845510483968","occurred_at":"2026-03-03T23:28:32Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T00:05:23.535Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T00:05:27.128Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T00:05:28.549Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=01e008b042f3b1d9d3583a6151c00926b2d82c157dbbd0a4a90f855cb1fcf047\ntime=2026-03-04T00:07:29.589Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0483968/providers/Microsoft.Resources/deployments/01e008b042f3b1d9d3583a6151c00926b2d82c157dbbd0a4a90f855cb1fcf047/operationStatuses/08584290241579070224\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0483968/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0483968/providers/Microsoft.Monitor/accounts/hcps-j0483968/providers/Microsoft.Authorization/roleAssignments/2fd1f45a-bf6a-5537-8079-b32100c7367b\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T000543Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '6e583ac6-8df7-4077-8554-590aca4bb7a9', request correlation id '7977b606-0c23-4359-a2f3-b9961854e5c8'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0483968/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0483968/providers/Microsoft.Monitor/accounts/services-j0483968/providers/Microsoft.Authorization/roleAssignments/7c700d51-06ff-53cd-9bca-75750e36f7fc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T000555Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'e7da9d64-1f4e-4665-a9dd-abd66fcb4339', request correlation id '7977b606-0c23-4359-a2f3-b9961854e5c8'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028967704882319360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T22:56:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028966994233004032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T22:53:22Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028966281062912000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T22:50:32Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4304/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028960408093593600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4304,"pr_state":"closed","pr_sha":"7dc387193a8168527a6670895d3fcf1d14b01184","final_merged_sha":"7dc387193a8168527a6670895d3fcf1d14b01184","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T22:27:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4287/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028957274038341632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4287,"pr_state":"closed","pr_sha":"723b6e68710379451b6a670f5df9529985329888","final_merged_sha":"723b6e68710379451b6a670f5df9529985329888","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T22:14:44Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6a6545da10e0247617954a75875a13c7059bf44b13b7bb3dc34bff2c78f02ac5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4287/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028957274038341632","occurred_at":"2026-03-03T22:14:44Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T22:43:22.218Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T22:43:24.140Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T22:43:25.336Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=b94e3dc6634aea5d22d910450ffc8aaac4c703ac5767f15be2142beb67bc9e83\ntime=2026-03-03T22:44:56.210Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8341632/providers/Microsoft.Resources/deployments/b94e3dc6634aea5d22d910450ffc8aaac4c703ac5767f15be2142beb67bc9e83/operationStatuses/08584290290808332756\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8341632/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8341632/providers/Microsoft.Monitor/accounts/services-j8341632/providers/Microsoft.Authorization/roleAssignments/1e51f5e3-3b1b-5e1a-810b-7839e7d0ede2\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T224347Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'b1f0c85d-dc9e-4a2e-b87d-233219182ab8', request correlation id '84d04789-5ae9-4b95-b28d-c46d7760476e'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8341632/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8341632/providers/Microsoft.Monitor/accounts/hcps-j8341632/providers/Microsoft.Authorization/roleAssignments/5a996a5c-bf08-52e4-bbce-288b41214d66\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T224356Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '3380b296-2d98-4ffb-8965-9aab0fb3cdc0', request correlation id '84d04789-5ae9-4b95-b28d-c46d7760476e'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028957135269793792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4290,"pr_state":"closed","pr_sha":"1710a252e4122ac0d277a20159eadd6f0506149b","final_merged_sha":"507eb199f7a4d7137d6ae5eddc82d001167895fb","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T22:14:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"622c780a812776f79b47545ad89040da8313a4ddb5e65e0360a0e8ce73ecf54d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028957135269793792","occurred_at":"2026-03-03T22:14:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028952232229081088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"bd26ee1d74a2c1a8fa1d0b631dc1027cddf4e49c","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T21:54:42Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"acef891d866cfdfe67e3ba1f240a8a2d55023736359ff99c68bd3cc9bd185dc9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028952232229081088","occurred_at":"2026-03-03T21:54:42Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T22:34:43.688Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T22:34:45.477Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T22:34:46.768Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8f1a43a3164993ff7a79d5fabd7f200f56738fa240fd68af115d3361b0993d9b\ntime=2026-03-03T22:36:17.605Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9081088/providers/Microsoft.Resources/deployments/8f1a43a3164993ff7a79d5fabd7f200f56738fa240fd68af115d3361b0993d9b/operationStatuses/08584290295995219744\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9081088/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9081088/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j9081088\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 4c16e0ed-e725-429f-b341-645816fd912b:3/3/2026 10:35:51 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9081088/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9081088/providers/Microsoft.Monitor/accounts/services-j9081088/providers/Microsoft.Authorization/roleAssignments/f7ea505e-032c-58f8-9c01-30dbb670142e\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T223506Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '8f2eec61-4537-4c20-b9ea-ea5729355e89', request correlation id '91d1922c-efcc-4ae0-94a3-9a4e7cc2c2ba'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9081088/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9081088/providers/Microsoft.Monitor/accounts/hcps-j9081088/providers/Microsoft.Authorization/roleAssignments/5c02a015-a8eb-58c3-9f82-6b559f9190e5\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T223508Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'c38338ed-74d8-45ce-8723-6099b9532542', request correlation id '91d1922c-efcc-4ae0-94a3-9a4e7cc2c2ba'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4299/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028943008090558464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4299,"pr_state":"closed","pr_sha":"196370357d407f49b404d5dbcf80e856958b5c8c","final_merged_sha":"89abb52bd0ca3634c8e64a287368b6c007666f1e","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-03T21:18:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4307/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028936121940971520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4307,"pr_state":"closed","pr_sha":"b1285555886dd6abd54876a17502ba490045124f","final_merged_sha":"b1285555886dd6abd54876a17502ba490045124f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T20:50:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028934181702078464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4258,"pr_state":"closed","pr_sha":"e2023dafc2500edee1f446eac424284b5b9df4e3","final_merged_sha":"e2023dafc2500edee1f446eac424284b5b9df4e3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T20:42:59Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4306/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028932144977416192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4306,"pr_state":"closed","pr_sha":"2918346a89d4cd9e705049820ba11246b6b2af7d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T20:34:53Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b8f41588c26d6c024e8844086023acc5a76941b53f3b0a8901f5d94a17622fed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4306/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028932144977416192","occurred_at":"2026-03-03T20:34:53Z","failed_at":"provision","signature_id":"7acbe25c7687742bceaa5fd499dcefdbc2e3cfd43241dff288a6f5013d29e221","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T21:21:01.840Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T21:21:04.229Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T21:21:05.527Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e07e02643917c7bd767a0273d7fdd7b255e1eb52a06f868df54a997be71cfb70\ntime=2026-03-03T21:22:36.238Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/e07e02643917c7bd767a0273d7fdd7b255e1eb52a06f868df54a997be71cfb70/operationStatuses/08584290340208132076\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/services-j7416192/providers/Microsoft.Authorization/roleAssignments/9175f7b5-343f-5dc0-ac51-0fc6bb9ff12d\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212120Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '2527100e-393a-4427-8595-0a8e3cce0b3c', request correlation id '9504cc96-8dad-4b88-a60f-7adfcdb5d3e6'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/hcps-j7416192/providers/Microsoft.Authorization/roleAssignments/fcbc90f9-43c6-578b-bdf9-8c870d5b0358\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212119Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'efa425fd-6da4-4b88-80f7-8127c106d790', request correlation id '9504cc96-8dad-4b88-a60f-7adfcdb5d3e6'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-03-03T21:24:36.246Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T21:24:37.070Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T21:24:38.123Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=70ddfe4f9da4df6335f66c29000b9a1deadaa5218866767d90b58a592c874b16\ntime=2026-03-03T21:25:08.480Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/70ddfe4f9da4df6335f66c29000b9a1deadaa5218866767d90b58a592c874b16/operationStatuses/08584290338079955644\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/hcps-j7416192/providers/Microsoft.Authorization/roleAssignments/fcbc90f9-43c6-578b-bdf9-8c870d5b0358\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212448Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'a8c40e7b-b1ed-4585-b265-e43f25c646be', request correlation id '845e4a38-7d08-487b-a22d-802b377de267'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-03-03T21:27:08.480Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T21:27:09.502Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T21:27:10.606Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5159deb33520cc62995efe45b56795fcf17e8ad2d0012150b793d02a5ddf558d\ntime=2026-03-03T21:27:40.993Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/5159deb33520cc62995efe45b56795fcf17e8ad2d0012150b793d02a5ddf558d/operationStatuses/08584290336555885853\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/hcps-j7416192/providers/Microsoft.Authorization/roleAssignments/fcbc90f9-43c6-578b-bdf9-8c870d5b0358\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212720Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '062e7270-48cd-4c68-b009-90783cc14a4a', request correlation id 'd2b40f1d-c3f6-4438-8b67-5401a0bd1298'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/services-j7416192/providers/Microsoft.Authorization/roleAssignments/9175f7b5-343f-5dc0-ac51-0fc6bb9ff12d\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212718Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'cf2d3d47-0527-40a7-81e1-a5c3f3762774', request correlation id 'd2b40f1d-c3f6-4438-8b67-5401a0bd1298'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-03-03T21:29:40.996Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/5159deb33520cc62995efe45b56795fcf17e8ad2d0012150b793d02a5ddf558d/operationStatuses/08584290336555885853\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/hcps-j7416192/providers/Microsoft.Authorization/roleAssignments/fcbc90f9-43c6-578b-bdf9-8c870d5b0358\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212720Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '062e7270-48cd-4c68-b009-90783cc14a4a', request correlation id 'd2b40f1d-c3f6-4438-8b67-5401a0bd1298'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7416192/providers/Microsoft.Monitor/accounts/services-j7416192/providers/Microsoft.Authorization/roleAssignments/9175f7b5-343f-5dc0-ac51-0fc6bb9ff12d\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T212718Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'cf2d3d47-0527-40a7-81e1-a5c3f3762774', request correlation id 'd2b40f1d-c3f6-4438-8b67-5401a0bd1298'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028921475443986432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T19:52:29Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028920752035598336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T19:49:37Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2e74009dc51e3d350c0cb0297342a0446183785f76e61b7bcac9ae37b1230894","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028920752035598336","occurred_at":"2026-03-03T19:49:37Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T20:31:16.404Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T20:31:19.304Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T20:31:20.722Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=63bd186d918b27a39ac36082a0114e90dbc0af57c42f6fd0171a5baf8e0f5774\ntime=2026-03-03T20:33:22.079Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5598336/providers/Microsoft.Resources/deployments/63bd186d918b27a39ac36082a0114e90dbc0af57c42f6fd0171a5baf8e0f5774/operationStatuses/08584290370055641106\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5598336/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5598336/providers/Microsoft.Monitor/accounts/hcps-j5598336/providers/Microsoft.Authorization/roleAssignments/7fff89ce-0e85-5a46-8759-4383616f3460\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T203143Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '6608d85f-6cdc-4d6c-bd77-6701dc5eb52d', request correlation id 'd8cf1b39-273a-46d9-95fb-12930873bc73'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5598336/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5598336/providers/Microsoft.Monitor/accounts/services-j5598336/providers/Microsoft.Authorization/roleAssignments/fd249894-2c98-522b-9711-098aba32ca02\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T203143Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '39584e29-af60-4c2e-b158-19dd5bca4915', request correlation id 'd8cf1b39-273a-46d9-95fb-12930873bc73'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028920026634915840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4254,"pr_state":"closed","pr_sha":"1ffff54f1d2e647ca0e3bb96102f12280f7605cf","final_merged_sha":"1ffff54f1d2e647ca0e3bb96102f12280f7605cf","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T19:46:44Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c85423c227e36794848b6129cb5b6fbc90e1372415c589ab5061412d54c6e0aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028920026634915840","occurred_at":"2026-03-03T19:46:44Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T20:15:28.523Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T20:15:31.304Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T20:15:32.986Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=0a6cdca4eae96aa66588596dc986fda8411bd731888567aca814dfe468cc9f68\ntime=2026-03-03T20:17:34.609Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4915840/providers/Microsoft.Resources/deployments/0a6cdca4eae96aa66588596dc986fda8411bd731888567aca814dfe468cc9f68/operationStatuses/08584290379534499269\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4915840/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4915840/providers/Microsoft.Monitor/accounts/hcps-j4915840/providers/Microsoft.Authorization/roleAssignments/cdc93feb-4d37-53e3-abcc-55be3d65534f\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T201547Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '50208b97-e613-4892-996f-15320177a2b6', request correlation id '98812432-cc4d-48c7-b0d1-8f9aac2aba39'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4304/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028903882469412864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4304,"pr_state":"closed","pr_sha":"7dc387193a8168527a6670895d3fcf1d14b01184","final_merged_sha":"7dc387193a8168527a6670895d3fcf1d14b01184","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T18:42:35Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"cdf7f40dfd93be69fbb5c07d72d01d2532a2470a5868793a232719c65eb06e56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4304/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028903882469412864","occurred_at":"2026-03-03T18:42:35Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T19:29:43.545Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T19:29:46.036Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T19:29:47.376Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f3cadb6faa13d93d2f684951d20d530263b365f9ad8316837a6cc0d6d8b298a3\ntime=2026-03-03T19:31:48.364Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9412864/providers/Microsoft.Resources/deployments/f3cadb6faa13d93d2f684951d20d530263b365f9ad8316837a6cc0d6d8b298a3/operationStatuses/08584290406989340272\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9412864/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9412864/providers/Microsoft.Monitor/accounts/hcps-j9412864/providers/Microsoft.Authorization/roleAssignments/59b9b767-f3a4-52b9-b2eb-dc7285af6e97\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T193000Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'aa35890a-a807-4e8d-b273-caecb0a3af34', request correlation id '30664faf-6aaa-48bc-b554-6adc279571cc'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028900085265338368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"8e69d4a5cd41631968bee3597f2cefd50484d545","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T18:27:29Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ee2cbb701eff51dbdf9dedc68c790b3e4da2f357d2ef7320f6c0579eb67b6337","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028900085265338368","occurred_at":"2026-03-03T18:27:29Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T18:58:21.952Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T18:58:24.041Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T18:58:25.349Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=efd59571860fd3ac9497a72fc4471c4dca54a459d1334ed42db56bb75f791b0e\ntime=2026-03-03T19:00:26.200Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5338368/providers/Microsoft.Resources/deployments/efd59571860fd3ac9497a72fc4471c4dca54a459d1334ed42db56bb75f791b0e/operationStatuses/08584290425809989599\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5338368/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5338368/providers/Microsoft.Monitor/accounts/services-j5338368/providers/Microsoft.Authorization/roleAssignments/6dc600af-646c-5041-b24d-ebc38c1a94d7\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T185905Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9938e214-6587-4997-b658-49dc2ed2c6c2', request correlation id 'db872278-62fe-4f8d-aade-232016bc84e7'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028897230248742912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4303,"pr_state":"closed","pr_sha":"7148a34a98c3daf4639ce203a1af6126324406ca","final_merged_sha":"7148a34a98c3daf4639ce203a1af6126324406ca","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T18:16:09Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"83bfacad0543d4556b1d016b50c7079088bc9bbdebf2c2fe285fa89ba6062256","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028897230248742912","occurred_at":"2026-03-03T18:16:09Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T18:44:20.855Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T18:44:23.490Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T18:44:24.831Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=29595863b20b733232a37c69860eb3c803694489e6ec9a8a4a9177d87db28ee1\ntime=2026-03-03T18:45:55.566Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8742912/providers/Microsoft.Resources/deployments/29595863b20b733232a37c69860eb3c803694489e6ec9a8a4a9177d87db28ee1/operationStatuses/08584290434214746991\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8742912/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8742912/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j8742912\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report ec8bf3a0-2358-4484-9b1d-22d51fdc1a15:3/3/2026 6:45:30 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4297/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028896789117014016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4297,"pr_state":"closed","pr_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","final_merged_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T18:14:23Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"50b74c276b90f57b86618fb62b2a667917ad2d61379835a3aba1dce5d49f4681","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4297/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028896789117014016","occurred_at":"2026-03-03T18:14:23Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T18:44:42.934Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T18:44:45.237Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T18:44:46.576Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=fc02f9b633ed78d5d4159063c0b4e8625410a37846ffe24018887df2a68cd2e5\ntime=2026-03-03T18:46:47.292Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7014016/providers/Microsoft.Resources/deployments/fc02f9b633ed78d5d4159063c0b4e8625410a37846ffe24018887df2a68cd2e5/operationStatuses/08584290433997349954\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7014016/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7014016/providers/Microsoft.Monitor/accounts/hcps-j7014016/providers/Microsoft.Authorization/roleAssignments/1f570be8-560e-5953-b86d-28353e7eecab\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T184527Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '1f262916-94cb-4aa3-80d5-797391d27614', request correlation id '9ffe5f6e-6832-4400-a6ac-38893fc423fb'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028894085338632192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4266,"pr_state":"closed","pr_sha":"b7c66ef63434e9282b921ed6b480eca0fa792d64","final_merged_sha":"7fbd7a365361285c704975d46eb93dffa3ec17d6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T18:03:39Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"baf82cc96e45529b53dfd30bfc5ef845997ce097f95ecf1fa00a14de007d3325","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028894085338632192","occurred_at":"2026-03-03T18:03:39Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T18:39:31.191Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T18:39:33.833Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T18:39:35.171Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=7cb5f23ee6789452609b27e401db327a0d78ac14e0046c37c7ea97592e0dcdd5\ntime=2026-03-03T18:41:36.432Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8632192/providers/Microsoft.Resources/deployments/7cb5f23ee6789452609b27e401db327a0d78ac14e0046c37c7ea97592e0dcdd5/operationStatuses/08584290437111508375\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8632192/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8632192/providers/Microsoft.Monitor/accounts/services-j8632192/providers/Microsoft.Authorization/roleAssignments/53fa87ca-b14e-5846-afce-e5d1e085c856\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T183958Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '2f0d908d-1954-4fa0-8ac7-8ba47fba6b35', request correlation id 'a1749b3a-86fd-46ca-a544-f592a7d6e79f'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028891133630746624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T17:51:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028890998343471104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T17:51:20Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"38a3e87d0fe6212b6d4973032815ce7099d72161a80124aaa4225570d4d53d64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028890998343471104","occurred_at":"2026-03-03T17:51:20Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T18:18:18.705Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T18:18:20.518Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T18:18:21.686Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=64a2b4246871719a5dd21c46273e2cbef883359f65435e8d586e48fb6533a080\ntime=2026-03-03T18:20:22.673Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j3471104/providers/Microsoft.Resources/deployments/64a2b4246871719a5dd21c46273e2cbef883359f65435e8d586e48fb6533a080/operationStatuses/08584290449844845101\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3471104/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3471104/providers/Microsoft.Monitor/accounts/services-j3471104/providers/Microsoft.Authorization/roleAssignments/457c2eff-bceb-52fb-b47d-f11e0dd381ff\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T181834Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'c120828d-c118-4c7d-9051-bd9340e543c2', request correlation id '8ae03719-a555-4fc3-86bc-5ba8899b3b86'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028890906379161600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T17:50:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8533524cfe480de6e0fbee6057512a64c8cfe17e588be522eb7efeafa19df04a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028890906379161600","occurred_at":"2026-03-03T17:50:56Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T18:24:32.908Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T18:24:35.306Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T18:24:36.417Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e69b7f69f0a6c15e18552e3b6ae80618311980544f643f32ac90d1ba590e77da\ntime=2026-03-03T18:26:07.320Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9161600/providers/Microsoft.Resources/deployments/e69b7f69f0a6c15e18552e3b6ae80618311980544f643f32ac90d1ba590e77da/operationStatuses/08584290446097598179\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9161600/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9161600/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j9161600\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report a546eb2b-9245-4986-b585-a26cef25fad7:3/3/2026 6:25:44 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9161600/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9161600/providers/Microsoft.Monitor/accounts/hcps-j9161600/providers/Microsoft.Authorization/roleAssignments/4fdfd6dd-e400-5775-a6ff-5e3947c1b047\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T182459Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'f13c9138-f83c-4d0e-924d-730676cd1827', request correlation id 'c43935a5-3026-40b8-9161-20dfd7560196'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4258/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028890659070414848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4258,"pr_state":"closed","pr_sha":"e2023dafc2500edee1f446eac424284b5b9df4e3","final_merged_sha":"e2023dafc2500edee1f446eac424284b5b9df4e3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T17:50:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028884866031947776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4259,"pr_state":"closed","pr_sha":"6f427a2276ceaf4cb0ac19b460588b2bfefed930","final_merged_sha":"6f427a2276ceaf4cb0ac19b460588b2bfefed930","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T17:27:01Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5d41bf48470585d2fe0f1e161cc2742deeb81622454b4ed1fc8835a26b2651da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028884866031947776","occurred_at":"2026-03-03T17:27:01Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028884062596239360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4266,"pr_state":"closed","pr_sha":"9c87d2e6656150cb025a9f85994226402da19f46","final_merged_sha":"7fbd7a365361285c704975d46eb93dffa3ec17d6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T17:23:49Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8930ccc937a620a88ca96fb60d09542b628855d32a24c8c7060114b6534d298f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028884062596239360","occurred_at":"2026-03-03T17:23:49Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028883664120582144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"open","pr_sha":"be9daeda83ecf50c0b263592b75cf782417142e9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-03T17:22:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028879957547028480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4290,"pr_state":"closed","pr_sha":"1710a252e4122ac0d277a20159eadd6f0506149b","final_merged_sha":"507eb199f7a4d7137d6ae5eddc82d001167895fb","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T17:07:31Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"dcc6c9ffc244a758a890706244b23bb14e1e93ec156a3b5949df7ae787ce406d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028879957547028480","occurred_at":"2026-03-03T17:07:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028868237260754944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4198,"pr_state":"open","pr_sha":"d931913b6b5570f95b2c5d7651a4b08b76c96ba5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T16:20:53Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3ddfd5b6010665d4d562deb8ae6b71ab6e6e850becfb9f6292f3986ae8ccfc35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028868237260754944","occurred_at":"2026-03-03T16:20:53Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T17:01:07.284Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T17:01:09.215Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T17:01:10.624Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9cecb097d9f924d2ffeabe37c43ecc0ea42d9b48407904871d4ca924494ea7f7\ntime=2026-03-03T17:03:11.784Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0754944/providers/Microsoft.Resources/deployments/9cecb097d9f924d2ffeabe37c43ecc0ea42d9b48407904871d4ca924494ea7f7/operationStatuses/08584290496157781617\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0754944/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0754944/providers/Microsoft.Monitor/accounts/services-j0754944/providers/Microsoft.Authorization/roleAssignments/c8c9d922-3ae1-5b66-990d-5c1f216992cd\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T170131Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'e58e5a43-759c-4acb-a70e-3989cd93c344', request correlation id 'b81c9b2e-6ac4-49ef-96e6-03e64583a706'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028867997694693376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4218,"pr_state":"closed","pr_sha":"cd42d690d61fe3044f9a6f43788fbc6776287e69","final_merged_sha":"7cbe20cfeaea24560965a517ee0964a9c659dbe3","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-03T16:19:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4254/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028867835375128576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4254,"pr_state":"closed","pr_sha":"1ffff54f1d2e647ca0e3bb96102f12280f7605cf","final_merged_sha":"1ffff54f1d2e647ca0e3bb96102f12280f7605cf","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T16:19:21Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4284/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028867644110671872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4284,"pr_state":"closed","pr_sha":"ea93c027290908107713053fe6ec653fe14058f2","final_merged_sha":"efd28c631288d18973d727e83e9a2b147a02894e","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-03T16:18:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4280/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028867189414563840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4280,"pr_state":"closed","pr_sha":"2106426157be13d22f3201da7f8224eda4ab860e","final_merged_sha":"2106426157be13d22f3201da7f8224eda4ab860e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T16:16:47Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028864097625313280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"open","pr_sha":"be9daeda83ecf50c0b263592b75cf782417142e9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T16:04:19Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c07cbaa83a3a23c04d0ff898c370b43851adfb2f41903e0aace0e2ad47b745ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028864097625313280","occurred_at":"2026-03-03T16:04:19Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T16:31:09.363Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T16:31:11.474Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T16:31:12.607Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=46b3ee3a87d785609c120b25686463796fe503721d34b32edda849c746d12d13\ntime=2026-03-03T16:33:13.523Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5313280/providers/Microsoft.Resources/deployments/46b3ee3a87d785609c120b25686463796fe503721d34b32edda849c746d12d13/operationStatuses/08584290514135610543\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5313280/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5313280/providers/Microsoft.Monitor/accounts/hcps-j5313280/providers/Microsoft.Authorization/roleAssignments/f0059e6b-f028-5cd7-9b37-a778c1d299f9\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T163147Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'c5bd731c-ecfd-4068-89b7-beddb4277bb0', request correlation id 'e583ca37-6ca2-4520-bc88-773589bee7c8'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4302/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028863384618799104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4302,"pr_state":"closed","pr_sha":"e5fc72e179142cd25b69e35c9cf72fe7f8b79601","final_merged_sha":"e5fc72e179142cd25b69e35c9cf72fe7f8b79601","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T16:01:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4289/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028858700201988096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4289,"pr_state":"closed","pr_sha":"2acf54cb98f656fdfbef25cb79ebd5af46e0b0f5","final_merged_sha":"f879b10dc600aa4bc317f4550e395d2e07047a53","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-03T15:43:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4271/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028858331761741824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4271,"pr_state":"closed","pr_sha":"70f3b0a98ca89b6e5529c731a62339633818d15e","final_merged_sha":"70f3b0a98ca89b6e5529c731a62339633818d15e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T15:41:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4273/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028858287260176384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4273,"pr_state":"closed","pr_sha":"0b6227fc7359d04685e6fa9b61c507ba2576a4fc","final_merged_sha":"0b6227fc7359d04685e6fa9b61c507ba2576a4fc","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T15:41:24Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4297/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028857985379340288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4297,"pr_state":"closed","pr_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","final_merged_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T15:40:12Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d239199ce88481140b3e66c4a22e9d9031f13d73f1c28224d3e651ba46b3871a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4297/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028857985379340288","occurred_at":"2026-03-03T15:40:12Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T16:11:03.728Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T16:11:05.607Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T16:11:06.938Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8ea32468eebea9aaa5b82e163d2a643cd5d4f81131257d37880093dc8c993f2d\ntime=2026-03-03T16:13:08.050Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9340288/providers/Microsoft.Resources/deployments/8ea32468eebea9aaa5b82e163d2a643cd5d4f81131257d37880093dc8c993f2d/operationStatuses/08584290526194167713\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9340288/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9340288/providers/Microsoft.Monitor/accounts/hcps-j9340288/providers/Microsoft.Authorization/roleAssignments/0592228e-12d3-535c-828e-9fba8d937713\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T161123Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7a52f29a-8e16-41e3-abf9-476ad3bd88c6', request correlation id '8fbcc270-16e8-4b47-b27b-eae93bf839a8'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4249,"pr_state":"closed","pr_sha":"67bc090ef1f250bde852b94552a8cfc7fe13e836","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T15:33:10Z"},"failed_at":["e2e"],"failed_test_count":14,"occurrences":[{"row_id":"326d2def8091c67067a61d635b436fe9b61cffb3f3fea2c791fea44cee04e499","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"5f7b26e38b337eceaf319121f5ce441be7e84ca4986dd80292103c9ca9802f03","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"0.1({0x7feb6c166e60, 0xc00072dec0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:80\n                | \n                | By(\"creating HCP cluster: \" + clusterName)\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc00072dec0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 91\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86 @ 03/03/26 18:06:58.653\n  \"ts\"=\"2026-03-03 18:06:58.654155\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-03 18:06:58.654256\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-list-test-cluster-k6k69k-5ctt57\" \"resourceGroup\"=\"cluster-listing-bm8xhc\"\n  \"ts\"=\"2026-03-03 18:06:59.535614\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-list-test-cluster-k6k69k-5ctt57\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-03 18:07:00.580421\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-list-test-cluster-9b5xbw-2hh42m\" \"resourceGroup\"=\"cluster-listing-4kc4lj\"\n  \"ts\"=\"2026-03-03 18:07:00.926424\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-list-test-cluster-9b5xbw-2hh42m\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-03 18:07:01.804167\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz\"\n  \"ts\"=\"2026-03-03 18:07:01.804285\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz\"\n  \"ts\"=\"2026-03-03 18:07:01.804350\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-03 18:07:01.805436\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-126\"\n  \"ts\"=\"2026-03-03 18:07:01.805442\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"cluster-listing-4kc4lj\"\n  \"ts\"=\"2026-03-03 18:07:01.805464\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"cluster-listing-bm8xhc\"\n  \"ts\"=\"2026-03-03 18:07:01.805525\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-32\"\n  \"ts\"=\"2026-03-03 18:07:02.407962\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-126\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-03 18:07:02.685913\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-bm8xhc\" \"deployment\"=\"pooledNonMsiScopedAssignments-kjvruuq77us5w\"\n  \"ts\"=\"2026-03-03 18:07:03.139911\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-4kc4lj\" \"deployment\"=\"pooledNonMsiScopedAssignments-erzs7xtv7toem\"\n  \"ts\"=\"2026-03-03 18:07:03.451923\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-32\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-03 18:07:03.863173\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-126\" \"deployment\"=\"msi-deployment-126\"\n  \"ts\"=\"2026-03-03 18:07:04.227778\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-bm8xhc\" \"deployment\"=\"customer-infra-list-test-cluster-k6k69k-5ctt57\"\n  \"ts\"=\"2026-03-03 18:07:04.769359\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-4kc4lj\" \"deployment\"=\"customer-infra-list-test-cluster-9b5xbw-2hh42m\"\n  \"ts\"=\"2026-03-03 18:07:05.030461\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-32\" \"deployment\"=\"msi-deployment-32\"\n  \"ts\"=\"2026-03-03 18:07:06.464726\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-03 18:07:06.464877\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-03 18:07:06.464997\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"cluster-listing-4kc4lj\"\n  \"ts\"=\"2026-03-03 18:07:06.465083\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"cluster-listing-bm8xhc\"\n  \"ts\"=\"2026-03-03 18:15:41.859301\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"cluster-listing-bm8xhc\"\n  \"ts\"=\"2026-03-03 18:15:41.859328\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"cluster-listing-bm8xhc\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n0.1({0x7feb6c166e60, 0xc00072dec0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:80\n                | \n                | By(\"creating HCP cluster: \" + clusterName)\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc00072dec0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 91\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86 @ 03/03/26 18:06:58.653\n  \"ts\"=\"2026-03-03 18:06:58.654155\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-03 18:06:58.654256\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-list-test-cluster-k6k69k-5ctt57\" \"resourceGroup\"=\"cluster-listing-bm8xhc\"\n  \"ts\"=\"2026-03-03 18:06:59.535614\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-list-test-cluster-k6k69k-5ctt57\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-03 18:07:00.580421\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-list-test-cluster-9b5xbw-2hh42m\" \"resourceGroup\"=\"cluster-listing-4kc4lj\"\n  \"ts\"=\"2026-03-03 18:07:00.926424\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-list-test-cluster-9b5xbw-2hh42m\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-03 18:07:01.804167\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz\"\n  \"ts\"=\"2026-03-03 18:07:01.804285\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz\"\n  \"ts\"=\"2026-03-03 18:07:01.804350\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-03 18:07:01.805436\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-126\"\n  \"ts\"=\"2026-03-03 18:07:01.805442\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"cluster-listing-4kc4lj\"\n  \"ts\"=\"2026-03-03 18:07:01.805464\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"cluster-listing-bm8xhc\"\n  \"ts\"=\"2026-03-03 18:07:01.805525\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-32\"\n  \"ts\"=\"2026-03-03 18:07:02.407962\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-126\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-03 18:07:02.685913\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-bm8xhc\" \"deployment\"=\"pooledNonMsiScopedAssignments-kjvruuq77us5w\"\n  \"ts\"=\"2026-03-03 18:07:03.139911\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-4kc4lj\" \"deployment\"=\"pooledNonMsiScopedAssignments-erzs7xtv7toem\"\n  \"ts\"=\"2026-03-03 18:07:03.451923\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-32\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-03 18:07:03.863173\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-126\" \"deployment\"=\"msi-deployment-126\"\n  \"ts\"=\"2026-03-03 18:07:04.227778\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-bm8xhc\" \"deployment\"=\"customer-infra-list-test-cluster-k6k69k-5ctt57\"\n  \"ts\"=\"2026-03-03 18:07:04.769359\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-4kc4lj\" \"deployment\"=\"customer-infra-list-test-cluster-9b5xbw-2hh42m\"\n  \"ts\"=\"2026-03-03 18:07:05.030461\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-32\" \"deployment\"=\"msi-deployment-32\"\n  \"ts\"=\"2026-03-03 18:07:06.464726\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-03 18:07:06.464877\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-03 18:07:06.464997\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"cluster-listing-4kc4lj\"\n  \"ts\"=\"2026-03-03 18:07:06.465083\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"cluster-listing-bm8xhc\"\n  \"ts\"=\"2026-03-03 18:15:41.859301\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"cluster-listing-bm8xhc\"\n  \"ts\"=\"2026-03-03 18:15:41.859328\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"cluster-listing-bm8xhc\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"455e71419ba6316e75809391131fb7ff09a63d3142a2ea4573e8c55897c31f11","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"02e3046f44567991bd187aeea634dc07e259c8c2031a3a26ed3b9e47aa1c7486","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ef4cf0\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-qgkfmt\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qgkfmt\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000e74810\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000ef4cc0\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qgkfmt\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000e74810\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ef4cf0\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-qgkfmt\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qgkfmt\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000e74810\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000ef4cc0\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qgkfmt\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000e74810\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-qgkfmt\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded"}},{"row_id":"5bd914778c056cec178ff4c2645aec080289c5b0702624809060b3a09dcdda9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"fe2ac3fd368abfd8752d0dc62b48e253a5c2e299e29913b89a64ab90ac66ba9b","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00057f740\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ljl5zz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ljl5zz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001048840\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00057f6b0\u003e{\n                msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ljl5zz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001048840\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00057f740\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ljl5zz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ljl5zz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001048840\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00057f6b0\u003e{\n                msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ljl5zz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001048840\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-ljl5zz\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded"}},{"row_id":"66894270f4190a74afd67bc494ec9e0b10eff063f260d01e6303a4f1a4f95ecd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"3c207f8f2d067594bb0e7c5d66a79dae07198e4f262bea475ea8598c9fe7a410","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e73e00\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-5mvz9d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-5mvz9d\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001223d20\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e73dd0\u003e{\n                msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-5mvz9d\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001223d20\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e73e00\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-5mvz9d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-5mvz9d\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001223d20\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e73dd0\u003e{\n                msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-5mvz9d\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001223d20\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5mvz9d\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded"}},{"row_id":"7ab8a2816ebb5bc816235caf01dafdf061c0797feeb980fbbb1943ee9a271282","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"648c57e60167085c8c5713249a2eadce885373550f6b5691cc6be44752445358","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a27050\u003e: \n    failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9whw8v\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-9whw8v\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000b11c60\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000a27020\u003e{\n                msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-9whw8v\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000b11c60\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a27050\u003e: \n    failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9whw8v\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-9whw8v\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000b11c60\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000a27020\u003e{\n                msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-9whw8v\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000b11c60\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-9whw8v\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded"}},{"row_id":"8496fbff8e336eff9ba79218f6d74fe8bf79b2d6e2841431356fd6d3a56eae47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"dfb90773cf0b51e4ea210bf650d989bcc6492181bd465faf0224db76ac9b9d13","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008c8e70\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-zg5nhq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-zg5nhq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0014e6640\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0008c8e40\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-zg5nhq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0014e6640\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008c8e70\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-zg5nhq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-zg5nhq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0014e6640\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0008c8e40\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-zg5nhq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0014e6640\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-zg5nhq\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded"}},{"row_id":"911a405cc3a146bbb635043dd47bfb8d572b26a8fade635e0e972697ae74c3ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"3a266ebbbc47415a446f2d479094971a31c2c2482560828f35b4ef30e089c325","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00141e990\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ltkkdm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-ltkkdm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000ffe1b0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00141e960\u003e{\n                msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-ltkkdm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000ffe1b0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00141e990\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ltkkdm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-ltkkdm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000ffe1b0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00141e960\u003e{\n                msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-ltkkdm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000ffe1b0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-ltkkdm\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded"}},{"row_id":"93a4a193834cdb8d4f0901e01387a487781ad1d32c5307261551407cff24efa4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"ed1f0e43e32f33d1179138b2f86468d29fda496bdc6ead34b76145e6570ba19e","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001552c30\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-2b5fxx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015a5970\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001552c00\u003e{\n                msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-2b5fxx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0015a5970\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001552c30\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-2b5fxx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015a5970\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001552c00\u003e{\n                msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-2b5fxx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0015a5970\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded"}},{"row_id":"99934c52fe025ab67ba52e625fee21419d3cca52a9fbb84e8f0ac010c82a50a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"c6cd3d51493e5ceaa474b35763f77f3d64b2ca6485120b628186360007542dc9","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001182a20\u003e: \n    failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-l2qgfq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-l2qgfq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00108cd60\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011829f0\u003e{\n                msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-l2qgfq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00108cd60\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001182a20\u003e: \n    failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-l2qgfq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-l2qgfq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00108cd60\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011829f0\u003e{\n                msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-l2qgfq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00108cd60\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-l2qgfq\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded"}},{"row_id":"b2acd957db0e593ca8f9631f238edbfc2747fc4afb1845c0c5f2e56566270567","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"d795e89316f3773ecad58cf74b17f5666e578639cba33d3f4e2ae5451a3c3191","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fa1740\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-6b59qg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-6b59qg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0012e1480\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000fa1710\u003e{\n                msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-6b59qg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0012e1480\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fa1740\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-6b59qg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-6b59qg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0012e1480\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000fa1710\u003e{\n                msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-6b59qg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0012e1480\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-6b59qg\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded"}},{"row_id":"ca798db6ed9eed1b5c5e10fd4dd162e3b2a1a15cc9afef121a78d5ea59005ae1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"6217d2b1a305d97a1c0af020752db2872039a8db3eb099ec48a297a5f6d8f549","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e22f60\u003e: \n    failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-btq5z7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-btq5z7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0017f1d30\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e22f30\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-btq5z7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0017f1d30\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e22f60\u003e: \n    failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-btq5z7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-btq5z7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0017f1d30\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e22f30\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-btq5z7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0017f1d30\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-btq5z7\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded"}},{"row_id":"dd5a64c55d027df57a7e6c5f2e5f637106aad2d17aaac3dc85476aa3f951ba68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"5db1a9bbebc794d06a29a3f42bab9d4394e48d8bcac460827ece0994c0bfe682","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"284\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7f1b192d5c60, 0xc00014cd20})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc00014cd20?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 105\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/03/26 18:06:58.671\n  \"ts\"=\"2026-03-03 18:06:58.672120\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-03 18:06:58.672190\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-xr92pd\" \"resourceGroup\"=\"customer-rg-7lt4zc\"\n  \"ts\"=\"2026-03-03 18:06:59.238010\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-xr92pd\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-03 18:07:00.144029\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-5qr5cd\" \"resourceGroup\"=\"customer-rg-7lt4zc\"\n  \"ts\"=\"2026-03-03 18:07:00.277203\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-5qr5cd\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-03 18:07:01.011697\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-03 18:07:01.011815\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-03 18:07:01.011908\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-03 18:07:01.012969\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-208\"\n  \"ts\"=\"2026-03-03 18:07:01.012992\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-279\"\n  \"ts\"=\"2026-03-03 18:07:01.013050\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-73\"\n  \"ts\"=\"2026-03-03 18:07:01.013077\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-7lt4zc\"\n  \"ts\"=\"2026-03-03 18:07:01.522360\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-208\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-03 18:07:02.020278\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-279\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-03 18:07:02.318172\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-73\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-03 18:07:02.771362\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-7lt4zc\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-03 18:07:03.100387\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-208\" \"deployment\"=\"msi-deployment-208\"\n  \"ts\"=\"2026-03-03 18:07:03.464514\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-279\" \"deployment\"=\"msi-deployment-279\"\n  \"ts\"=\"2026-03-03 18:07:03.817218\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-73\" \"deployment\"=\"msi-deployment-73\"\n  \"ts\"=\"2026-03-03 18:07:04.185626\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-7lt4zc\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-5qr5cd\"\n  \"ts\"=\"2026-03-03 18:07:05.686924\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-7lt4zc\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-03 18:07:06.242343\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-7lt4zc\" \"deployment\"=\"customer-infra-basic-hcp-cluster-xr92pd\"\n  \"ts\"=\"2026-03-03 18:07:06.716603\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-03 18:07:06.716740\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-03 18:07:06.716879\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-7lt4zc\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n284\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7f1b192d5c60, 0xc00014cd20})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc00014cd20?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 105\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/03/26 18:06:58.671\n  \"ts\"=\"2026-03-03 18:06:58.672120\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-03 18:06:58.672190\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-xr92pd\" \"resourceGroup\"=\"customer-rg-7lt4zc\"\n  \"ts\"=\"2026-03-03 18:06:59.238010\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-xr92pd\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-03 18:07:00.144029\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-5qr5cd\" \"resourceGroup\"=\"customer-rg-7lt4zc\"\n  \"ts\"=\"2026-03-03 18:07:00.277203\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-5qr5cd\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-03 18:07:01.011697\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-03 18:07:01.011815\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-03 18:07:01.011908\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-03 18:07:01.012969\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-208\"\n  \"ts\"=\"2026-03-03 18:07:01.012992\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-279\"\n  \"ts\"=\"2026-03-03 18:07:01.013050\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-73\"\n  \"ts\"=\"2026-03-03 18:07:01.013077\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-7lt4zc\"\n  \"ts\"=\"2026-03-03 18:07:01.522360\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-208\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-03 18:07:02.020278\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-279\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-03 18:07:02.318172\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-73\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-03 18:07:02.771362\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-7lt4zc\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-03 18:07:03.100387\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-208\" \"deployment\"=\"msi-deployment-208\"\n  \"ts\"=\"2026-03-03 18:07:03.464514\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-279\" \"deployment\"=\"msi-deployment-279\"\n  \"ts\"=\"2026-03-03 18:07:03.817218\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-73\" \"deployment\"=\"msi-deployment-73\"\n  \"ts\"=\"2026-03-03 18:07:04.185626\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-7lt4zc\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-5qr5cd\"\n  \"ts\"=\"2026-03-03 18:07:05.686924\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-7lt4zc\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-03 18:07:06.242343\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-7lt4zc\" \"deployment\"=\"customer-infra-basic-hcp-cluster-xr92pd\"\n  \"ts\"=\"2026-03-03 18:07:06.716603\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-03 18:07:06.716740\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-03 18:07:06.716879\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-7lt4zc\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"e772d5ea0cde1deb3c9d505ada35ba75e48ab22c5159c7c33aedd7b840e32735","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"79ec8f702cc90d84c90e475b55b0dd66ebb90cc65938422146654600a0096f32","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015d83c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-45f8zm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-45f8zm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015a6a60\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0015d8390\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-45f8zm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0015a6a60\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015d83c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-45f8zm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-45f8zm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015a6a60\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0015d8390\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-45f8zm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0015a6a60\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-45f8zm\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded"}},{"row_id":"f4bc292da163f836f02e6830d6eef2e557f323e34b0e5d4028d72163aa09d63b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028856213021659136","occurred_at":"2026-03-03T15:33:10Z","failed_at":"e2e","signature_id":"0e1049ce83d4f51cbb2dc7d55aae6d22e3a14a0a91fc4c7aa4025a38bb164ec4","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012cac60\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-nt95fb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nt95fb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001297c90\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0012cac30\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nt95fb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001297c90\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012cac60\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-nt95fb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nt95fb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001297c90\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0012cac30\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nt95fb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001297c90\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nt95fb\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2b5fxx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2b5fxx, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":14,"failure_pattern_ids":["9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":14,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4062/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028855079292899328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4062,"pr_state":"closed","pr_sha":"26b2f214e2d6443a2d37ec7705c6a64a65050f60","final_merged_sha":"26b2f214e2d6443a2d37ec7705c6a64a65050f60","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T15:28:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028853366318174208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4292,"pr_state":"closed","pr_sha":"c28b827e7f1371d1cfdc2d230c0478ccf30c5372","final_merged_sha":"c28b827e7f1371d1cfdc2d230c0478ccf30c5372","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-03T15:21:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028836238877790208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4292,"pr_state":"closed","pr_sha":"c28b827e7f1371d1cfdc2d230c0478ccf30c5372","final_merged_sha":"c28b827e7f1371d1cfdc2d230c0478ccf30c5372","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T14:13:47Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"75e6e1241f0e85c34e0b47375b962355c618e9531c6a5897fd02168327fb7420","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028836238877790208","occurred_at":"2026-03-03T14:13:47Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T14:46:05.987Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T14:46:08.124Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T14:46:09.313Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d1a47a3051f2c857f26b8adf5743ab7f600aa055b7522f13b681251f97dfba4b\ntime=2026-03-03T14:47:40.143Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7790208/providers/Microsoft.Resources/deployments/d1a47a3051f2c857f26b8adf5743ab7f600aa055b7522f13b681251f97dfba4b/operationStatuses/08584290577168944907\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7790208/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7790208/providers/Microsoft.Monitor/accounts/services-j7790208/providers/Microsoft.Authorization/roleAssignments/95ec45db-bb4e-515b-961d-153f8b7e7e47\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T144630Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '610d3bc0-a518-456f-87aa-0c7af69f86aa', request correlation id '3cd1d246-f469-4cad-8013-bd7ad88caf91'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028835456589762560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4181,"pr_state":"closed","pr_sha":"e8fec6bb18b616d66d477e112301f58adcdc2bcd","final_merged_sha":"0ac5e198ed7921a253ae14ad2c7ea366e71f81ce","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-03T14:10:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028834973850537984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4249,"pr_state":"closed","pr_sha":"67bc090ef1f250bde852b94552a8cfc7fe13e836","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T14:08:46Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"128411db491a61230621833828d1f2fa892b427f9b59e02a106e5d3ca060932e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028834973850537984","occurred_at":"2026-03-03T14:08:46Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T14:40:13.311Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T14:40:15.781Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T14:40:17.253Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=3b23317ac96742e5ffa43a7dfc39860a846f8fffe6e216de8ee57eacc9589cd7\ntime=2026-03-03T14:41:48.100Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0537984/providers/Microsoft.Resources/deployments/3b23317ac96742e5ffa43a7dfc39860a846f8fffe6e216de8ee57eacc9589cd7/operationStatuses/08584290580690595756\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0537984/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0537984/providers/Microsoft.Monitor/accounts/services-j0537984/providers/Microsoft.Authorization/roleAssignments/0ffa2d13-e94b-5d19-8aee-3f3a85caa948\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T144031Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '21a4d2da-0b47-4f56-ad1f-2fdffde5755a', request correlation id 'e5b1cb59-2756-424b-85c4-3d0c9ba186c5'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4279/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028833423077937152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4279,"pr_state":"closed","pr_sha":"9bce19c0b253a1664b88a85024fa247529209aff","final_merged_sha":"efa8e746c645a87051488818bbf21f67fb2329fd","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-03T14:02:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028831168715034624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4290,"pr_state":"closed","pr_sha":"1710a252e4122ac0d277a20159eadd6f0506149b","final_merged_sha":"507eb199f7a4d7137d6ae5eddc82d001167895fb","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T13:53:39Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f4ad71f5298a55b51074674f563c4414a79baa65afa25e09e73c50b6276f3040","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028831168715034624","occurred_at":"2026-03-03T13:53:39Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4062/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028826432125276160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4062,"pr_state":"closed","pr_sha":"26b2f214e2d6443a2d37ec7705c6a64a65050f60","final_merged_sha":"26b2f214e2d6443a2d37ec7705c6a64a65050f60","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T13:34:49Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0a9ae158ca2966a00461aa4f3ca3d8b4ddb843a011046f518d660b24e20b1ed5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4062/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028826432125276160","occurred_at":"2026-03-03T13:34:49Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T14:02:53.493Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T14:02:55.987Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T14:02:57.399Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=108f2eb68399ed8c24881221a7ea419480bbc39de21cb695ec1c25999c49a08b\ntime=2026-03-03T14:04:28.175Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5276160/providers/Microsoft.Resources/deployments/108f2eb68399ed8c24881221a7ea419480bbc39de21cb695ec1c25999c49a08b/operationStatuses/08584290603088964675\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5276160/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5276160/providers/Microsoft.Monitor/accounts/hcps-j5276160/providers/Microsoft.Authorization/roleAssignments/aa532dc4-e19b-50a3-99c9-65704d740d8e\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T140311Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '8dae1e94-3d2d-49cb-ba0f-43151072c8f5', request correlation id 'a8364498-ca2e-4dba-a4a6-d8290b38b749'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5276160/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5276160/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j5276160\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 492d1aa0-cd90-4eb7-af4f-b0c2bcbcaaa1:3/3/2026 2:04:11 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5276160/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5276160/providers/Microsoft.Monitor/accounts/services-j5276160/providers/Microsoft.Authorization/roleAssignments/d07bec58-1bd9-5ce8-9e2e-ddad3342714f\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T140319Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '1d90f6eb-e8ff-48ef-910c-f85482c765fc', request correlation id 'a8364498-ca2e-4dba-a4a6-d8290b38b749'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4299/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028825424045281280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4299,"pr_state":"closed","pr_sha":"5035eeb5ac2e16993775fb318276de20704371cd","final_merged_sha":"89abb52bd0ca3634c8e64a287368b6c007666f1e","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-03T13:30:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3760/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028821487363821568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3760,"pr_state":"open","pr_sha":"12f8f995c2909a2f0abe8184017fe37fe172e5c0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T13:15:10Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0100ca5e9b8176562734bab5cba3354ee7fb8499acaa9e418491ff0e202dbadd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3760/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028821487363821568","occurred_at":"2026-03-03T13:15:10Z","failed_at":"e2e","signature_id":"a90b7002c8c15e3f5ee97fd16e2258a207d55eb9bd5bfc3cfd94f18d51fcccb0","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e283c0\u003e: \n    failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n    {\n        msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n        err: \u003c*errors.StatusError | 0xc000b30960\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n                Reason: \"NotFound\",\n                Details: {\n                    Name: \"aro-hcp-exporter\",\n                    Group: \"\",\n                    Kind: \"services\",\n                    UID: \"\",\n                    Causes: nil,\n                    RetryAfterSeconds: 0,\n                },\n                Code: 404,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e283c0\u003e: \n    failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n    {\n        msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n        err: \u003c*errors.StatusError | 0xc000b30960\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n                Reason: \"NotFound\",\n                Details: {\n                    Name: \"aro-hcp-exporter\",\n                    Group: \"\",\n                    Kind: \"services\",\n                    UID: \"\",\n                    Causes: nil,\n                    RetryAfterSeconds: 0,\n                },\n                Code: 404,\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308","failure_pattern":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found","search_query":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4297/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028819096543432704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4297,"pr_state":"closed","pr_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","final_merged_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T13:05:40Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0250ba64d098b8c423907eae303ea7f15c1d6487b36d8dff752fa466caba51e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4297/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028819096543432704","occurred_at":"2026-03-03T13:05:40Z","failed_at":"e2e","signature_id":"0ff62bba9c8c6ce875f22557691eeb0cbe2d2b311fc75f5ff9f54c14b0867526","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00090f2a0\u003e: \n    failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n    {\n        msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n        err: \u003c*errors.StatusError | 0xc000c660a0\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n                Reason: \"NotFound\",\n                Details: {\n                    Name: \"aro-hcp-exporter\",\n                    Group: \"\",\n                    Kind: \"services\",\n                    UID: \"\",\n                    Causes: nil,\n                    RetryAfterSeconds: 0,\n                },\n                Code: 404,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00090f2a0\u003e: \n    failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n    {\n        msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n        err: \u003c*errors.StatusError | 0xc000c660a0\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n                Reason: \"NotFound\",\n                Details: {\n                    Name: \"aro-hcp-exporter\",\n                    Group: \"\",\n                    Kind: \"services\",\n                    UID: \"\",\n                    Causes: nil,\n                    RetryAfterSeconds: 0,\n                },\n                Code: 404,\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308","failure_pattern":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found","search_query":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4062/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028812704579325952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4062,"pr_state":"closed","pr_sha":"26b2f214e2d6443a2d37ec7705c6a64a65050f60","final_merged_sha":"26b2f214e2d6443a2d37ec7705c6a64a65050f60","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T12:40:16Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ca23d678f5b4cc62bcf6ddbeaaa4683f6b7db93aa1d61b3f36bf3028439e75dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4062/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028812704579325952","occurred_at":"2026-03-03T12:40:16Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T13:07:03.306Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T13:07:05.149Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T13:07:06.504Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=594af2ff45a2dfb34b82fdf9cf8bc25823cfaf32f3cd653dd64fc00e328d64fc\ntime=2026-03-03T13:08:37.297Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9325952/providers/Microsoft.Resources/deployments/594af2ff45a2dfb34b82fdf9cf8bc25823cfaf32f3cd653dd64fc00e328d64fc/operationStatuses/08584290636598606279\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9325952/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9325952/providers/Microsoft.Monitor/accounts/services-j9325952/providers/Microsoft.Authorization/roleAssignments/0575d068-f21c-5b7f-8844-1eddcc2174b9\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T130727Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '0f4b6721-09c6-429b-96af-8b552889a7c5', request correlation id '203770a4-a8cc-4bb2-81d0-a53a6cde2aa7'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028810084603138048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4278,"pr_state":"closed","pr_sha":"ae3db774ae1652c8f7aa816af31b41f54645f19b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-03T12:29:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028807960259465216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4075,"pr_state":"closed","pr_sha":"e6bfec028f5a8081ea1e6eead9bb9c6ba8c903e8","final_merged_sha":"4c157031af423b1f7a6117f86f65c7962bd89ff5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T12:21:25Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6f982221a20be828539d2cb5005a9475f15c01a5182c99b059d58eaf950cb70e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028807960259465216","occurred_at":"2026-03-03T12:21:25Z","failed_at":"e2e","signature_id":"6414332284eb576304fce979f0fd175870a02f8303e1e64dd5d482ee7b3eac21","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000587040\u003e: \n    failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n    {\n        msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n        err: \u003c*errors.StatusError | 0xc000b62780\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n                Reason: \"NotFound\",\n                Details: {\n                    Name: \"aro-hcp-exporter\",\n                    Group: \"\",\n                    Kind: \"services\",\n                    UID: \"\",\n                    Causes: nil,\n                    RetryAfterSeconds: 0,\n                },\n                Code: 404,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000587040\u003e: \n    failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n    {\n        msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n        err: \u003c*errors.StatusError | 0xc000b62780\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n                Reason: \"NotFound\",\n                Details: {\n                    Name: \"aro-hcp-exporter\",\n                    Group: \"\",\n                    Kind: \"services\",\n                    UID: \"\",\n                    Causes: nil,\n                    RetryAfterSeconds: 0,\n                },\n                Code: 404,\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308","failure_pattern":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found","search_query":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4273/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028803479492890624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4273,"pr_state":"closed","pr_sha":"0b6227fc7359d04685e6fa9b61c507ba2576a4fc","final_merged_sha":"0b6227fc7359d04685e6fa9b61c507ba2576a4fc","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T12:03:37Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1eeab687e9b24961fd853733950b872cc3c62a948e84054157e63d9c83863a5a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4273/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028803479492890624","occurred_at":"2026-03-03T12:03:37Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T12:31:07.922Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T12:31:09.457Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T12:31:10.642Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cd4b7faf105951b179be587d1cc2d3817259d713c2a4ee210d4639f3dfe2cde7\ntime=2026-03-03T12:32:41.561Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j2890624/providers/Microsoft.Resources/deployments/cd4b7faf105951b179be587d1cc2d3817259d713c2a4ee210d4639f3dfe2cde7/operationStatuses/08584290658155093078\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2890624/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2890624/providers/Microsoft.Monitor/accounts/services-j2890624/providers/Microsoft.Authorization/roleAssignments/f674afde-a7be-58ca-a533-6629e33eed66\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T123135Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '99ff43a2-da4c-4e95-907d-ef58695df9d9', request correlation id 'af35b212-71d3-4d4e-9197-3c75f7302405'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4205/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028799376427585536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4205,"pr_state":"closed","pr_sha":"8598d2daf92c889c440f07def4bfb97e6ff150f9","final_merged_sha":"a36f37aae6ead63c15a10186a904f97e4e0fc50d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T11:47:19Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3b88666c2b91864bd9d9d7530dbed44b0a03946608bc9857e471db6c4f1dfd5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4205/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028799376427585536","occurred_at":"2026-03-03T11:47:19Z","failed_at":"e2e","signature_id":"972860d5340f65b2cc4b440c34865c875a358e9da722d878bd6bc8101e1c3105","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f82120\u003e: \n    failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n    {\n        msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n        err: \u003c*errors.StatusError | 0xc000f94960\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n                Reason: \"NotFound\",\n                Details: {\n                    Name: \"aro-hcp-exporter\",\n                    Group: \"\",\n                    Kind: \"services\",\n                    UID: \"\",\n                    Causes: nil,\n                    RetryAfterSeconds: 0,\n                },\n                Code: 404,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f82120\u003e: \n    failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n    {\n        msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n        err: \u003c*errors.StatusError | 0xc000f94960\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n                Reason: \"NotFound\",\n                Details: {\n                    Name: \"aro-hcp-exporter\",\n                    Group: \"\",\n                    Kind: \"services\",\n                    UID: \"\",\n                    Causes: nil,\n                    RetryAfterSeconds: 0,\n                },\n                Code: 404,\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308","failure_pattern":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found","search_query":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4289/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028783558616682496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4289,"pr_state":"closed","pr_sha":"2acf54cb98f656fdfbef25cb79ebd5af46e0b0f5","final_merged_sha":"f879b10dc600aa4bc317f4550e395d2e07047a53","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T10:44:26Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d2a56be05d89f62d36b1f34a9ae98eab9345aadec2639b63ab09d767d66e1b2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4289/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028783558616682496","occurred_at":"2026-03-03T10:44:26Z","failed_at":"e2e","signature_id":"5dc7500b99d8c12b2768cd057bcab36d69a2aebc1eb1dc4a8a24d0fdc2c87a9f","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7c8c0\u003e: \n    failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n    {\n        msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n        err: \u003c*errors.StatusError | 0xc0009fa460\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n                Reason: \"NotFound\",\n                Details: {\n                    Name: \"aro-hcp-exporter\",\n                    Group: \"\",\n                    Kind: \"services\",\n                    UID: \"\",\n                    Causes: nil,\n                    RetryAfterSeconds: 0,\n                },\n                Code: 404,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7c8c0\u003e: \n    failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n    {\n        msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n        err: \u003c*errors.StatusError | 0xc0009fa460\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n                Reason: \"NotFound\",\n                Details: {\n                    Name: \"aro-hcp-exporter\",\n                    Group: \"\",\n                    Kind: \"services\",\n                    UID: \"\",\n                    Causes: nil,\n                    RetryAfterSeconds: 0,\n                },\n                Code: 404,\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308","failure_pattern":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found","search_query":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028775000722378752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4075,"pr_state":"closed","pr_sha":"5b6dfb3bedbac59d4ee550297b8954f450ee93c4","final_merged_sha":"4c157031af423b1f7a6117f86f65c7962bd89ff5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T10:10:27Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"eb10381d4847971c585347126adad6bdd7df9e4ca907622a20aeca69baabd5b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028775000722378752","occurred_at":"2026-03-03T10:10:27Z","failed_at":"provision","signature_id":"385473de5e27e801ee8259d3aa8cb0addc6bce1417e4079d15464be8757946df","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-03T11:15:46.995Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-03T11:15:47.729Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2378752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2378752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae databaseName:arohcpprow-rp-j2378752 databaseUrl:https://arohcpprow-rp-j2378752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-pqkfnvp1/pipeline@sha256:a5bee0558fd61a844225e269b6a1d9a4b193cc74e2d161889b49783f729e214e replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-pqkfnvp1/pipeline:pull workloadIdentityClientId:52b2eb73-d9ac-4182-b194-6b3e19839ae0 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-03T11:15:47.729Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:15:48.047Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-03T11:15:48.459Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-03T11:15:48.459Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:15:48.629Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:20:54.787Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-03T11:20:54.787Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:20:54.787Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:20:59.078Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2378752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2378752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae databaseName:arohcpprow-rp-j2378752 databaseUrl:https://arohcpprow-rp-j2378752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-pqkfnvp1/pipeline@sha256:a5bee0558fd61a844225e269b6a1d9a4b193cc74e2d161889b49783f729e214e replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-pqkfnvp1/pipeline:pull workloadIdentityClientId:52b2eb73-d9ac-4182-b194-6b3e19839ae0 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-03T11:20:59.080Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4xTwW4aMRS871c87SlI6whIU7VUOaRpT1Uq1PSUqgevPQUHr9/q2Qsi4uMrQykrBC231XjmjWefp/BIJIjciUGkO7I66aRrjyuuX2DSFxfsJCZxYVbRDvqqG/yFsETYIrHVZg8PfhREROUTZOkM7o3hLqSyorLWZoFg86cWVnPTltWO+409PrpgXZj1iMpDW4iCh0mOwwnhH5PeidqLGyRxJp4Q3RuZdt6fMGw771V9wI+FXZqzuFedbzNl78x6y/KeV3s/dT7lE4wgTYWXzkIevI6xbx53WdQCa7XUnU/n8z5ycInlROwTmgcOv9zsUbd9M/3aCZR0IbkGymwpl2rPsvMaM/z/vX1C63ndIKTLMkwByX8fITmjj4b+a+XFzw/FoqshAQnxc36wsdjQag4B5ewx6aalGmkFBLqyOiHDV+V4OH6rhjdqePN9NJqMbidv3j2XA7q+pnOc8XBy+/65HAyKDb2wC7Rwwd65ECC9mnGgQ7v6tTruU7GhVjifHi5aUct2R75sSEWeZxU1iFHP8kgWC6F63QtvEc3vAAAA///4hORaDAQAAA==\"\ntime=2026-03-03T11:20:59.082Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:20:59.082Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-03T11:21:59.083Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-03T11:21:59.681Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2378752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2378752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae databaseName:arohcpprow-rp-j2378752 databaseUrl:https://arohcpprow-rp-j2378752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-pqkfnvp1/pipeline@sha256:a5bee0558fd61a844225e269b6a1d9a4b193cc74e2d161889b49783f729e214e replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-pqkfnvp1/pipeline:pull workloadIdentityClientId:52b2eb73-d9ac-4182-b194-6b3e19839ae0 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-03T11:21:59.681Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:22:00.052Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-03T11:22:00.215Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:22:00.216Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:22:00.377Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-03-03T11:22:01.435Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:22:01.435Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=aro-hcp name=backend\ntime=2026-03-03T11:22:01.734Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-azure-runtime-config\ntime=2026-03-03T11:22:02.176Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-config\ntime=2026-03-03T11:22:02.341Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=aro-hcp name=leader-election\ntime=2026-03-03T11:22:02.999Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=aro-hcp name=backend-leader-election\ntime=2026-03-03T11:22:03.303Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-03T11:22:03.628Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-03T11:22:03.968Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=aro-hcp name=backend-pull-binding\ntime=2026-03-03T11:22:04.324Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=aro-hcp name=allow-metrics-backend\ntime=2026-03-03T11:22:04.663Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-03T11:22:05.112Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=aro-hcp name=backend-service-key-vault\ntime=2026-03-03T11:22:05.410Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-03T11:22:05.778Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-03T11:22:05.778Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:22:05.893Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-03-03T11:27:08.959Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-03T11:27:08.959Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:27:08.959Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:27:12.870Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2378752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2378752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae databaseName:arohcpprow-rp-j2378752 databaseUrl:https://arohcpprow-rp-j2378752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-pqkfnvp1/pipeline@sha256:a5bee0558fd61a844225e269b6a1d9a4b193cc74e2d161889b49783f729e214e replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-pqkfnvp1/pipeline:pull workloadIdentityClientId:52b2eb73-d9ac-4182-b194-6b3e19839ae0 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-03T11:27:12.872Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4xTTW8aMRC9768Y7SlI6wiI2kpUOaRpT1Uq1PaUqgev/QoOXs9q7AUR8eMrQykrtERc37z35kPzCo9EgsidGES6J6uTTrr2uOH6BSZ9dcHOYhIXFhUdoG+6wX8Ia4Q9ElttjvDoV0FEVH5nj08uWBcWZUVlrc0KwSoPbSEKHiY5DrmkhdXStGV1EP6AEaS58NpZyKPXMfYNImTtDNQKW7XWnU+DFnvOgzHchdRTX+b2KurYqkESZ+KA6DNaz9sGIQ3oBvgPRuad9wP3aDvvVX3Cz4VdWrK4V52PNWfvzHbP8p43x/ne6DsHJFsgJGf02cGv2PPfcZ44uMRy3a6PHP64xZNu+2vq106gpAvJNVBmT7lWe5GdXyzDb/xU8ftjsepqSEBC/JIfNhY72iwhoDxLTLppqUbaAIFurE7I8E05HU/fq/GdGt/9nExm0+ls/O65HNHtLV3kfJhNps/laFTs6IVdoJUL9t6FAOnFjAOd0tWP1Xmeih21wrl6GrSilu2BfJ1JRZ4XFTWIUS+yJYuFUL3tLW8Rzd8AAAD//3NXtwQMBAAA\"\ntime=2026-03-03T11:27:12.875Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:27:12.875Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-03T11:28:12.880Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-03T11:28:13.896Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2378752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2378752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae databaseName:arohcpprow-rp-j2378752 databaseUrl:https://arohcpprow-rp-j2378752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-pqkfnvp1/pipeline@sha256:a5bee0558fd61a844225e269b6a1d9a4b193cc74e2d161889b49783f729e214e replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-pqkfnvp1/pipeline:pull workloadIdentityClientId:52b2eb73-d9ac-4182-b194-6b3e19839ae0 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-03T11:28:13.896Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:28:14.240Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-03T11:28:14.471Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:28:14.471Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:28:14.631Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-03-03T11:28:15.790Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:28:15.791Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=aro-hcp name=backend\ntime=2026-03-03T11:28:16.106Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-azure-runtime-config\ntime=2026-03-03T11:28:16.459Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-config\ntime=2026-03-03T11:28:16.631Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=aro-hcp name=leader-election\ntime=2026-03-03T11:28:17.182Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=aro-hcp name=backend-leader-election\ntime=2026-03-03T11:28:17.524Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-03T11:28:17.898Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-03T11:28:18.302Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=aro-hcp name=backend-pull-binding\ntime=2026-03-03T11:28:18.595Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=aro-hcp name=allow-metrics-backend\ntime=2026-03-03T11:28:19.133Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-03T11:28:19.565Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=aro-hcp name=backend-service-key-vault\ntime=2026-03-03T11:28:19.982Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-03T11:28:20.327Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-03T11:28:20.327Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:28:20.498Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-03-03T11:33:23.598Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-03T11:33:23.598Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:33:23.599Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:33:28.035Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2378752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2378752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae databaseName:arohcpprow-rp-j2378752 databaseUrl:https://arohcpprow-rp-j2378752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-pqkfnvp1/pipeline@sha256:a5bee0558fd61a844225e269b6a1d9a4b193cc74e2d161889b49783f729e214e replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-pqkfnvp1/pipeline:pull workloadIdentityClientId:52b2eb73-d9ac-4182-b194-6b3e19839ae0 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:66a3d10d-4291-4b20-8191-aa3e5a1300ae workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-03T11:33:28.037Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5STUWvbPBSG7/0rDr5KwCppAh8lH73Iul2NjrDtqmMXsvQuUSPrmCM5ISU/ftiZF1Ncmt2+ep5XluSTeSQSRG7EINI9WZ100qXHhMtnmPTZBbuMSVzYFHSOvugKfyPsEbok1tr08fRHRkSUr5q0ZXEvOjkOa/bOHPOCcu09H1SFJM5EVWqzQ7DdgrDamjovzvoakLYCITnTdQyg3ut7RvxvMIK0Ft47C3nwOnZUL0bI3hmoHY5qrxufRis65pGDSywj2484Dxx+uc2jroeb6ZdGoKQJyVVQpkOudd+kv7JHG3toC1HwMK9uaUh+cMG6sBk2vy9+RO35WCGk6w7/58JWxnBzdt5l/+1RV0bWjfcjh6kb71V5yXsx+/l/tmtKSEBC/NT+sDE70WELAbXPEZOuaiqRDkCgidUJbTzJ57P5f2q2ULPF99vb5fxuOZ895VO6uaG3mMViOb97yqfT7ETP7ALtXLD3LgTIYMw40GW6hmP1ep6yE9XC7erlQwuq2Z7h60oK8rwpqEKMetNWslgIlcfB4S2i+R0AAP//2E6uVAwEAAA=\"\ntime=2026-03-03T11:33:28.039Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-03T11:33:28.039Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-03T11:34:28.039Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9a3364ebecf8b32a7511e0c1a2f7a3bf0739c1b446124dca86eaa8a33f7d1fa","failure_pattern":"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d9a3364ebecf8b32a7511e0c1a2f7a3bf0739c1b446124dca86eaa8a33f7d1fa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4279/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028772421699375104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4279,"pr_state":"closed","pr_sha":"2370bdf2ecf300a52b3702f979da0a2455305ce3","final_merged_sha":"efa8e746c645a87051488818bbf21f67fb2329fd","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T10:00:12Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b5d4da48406f70bb1fc795582626685dd879423d4f008f594bab115d3a0e6546","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4279/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028772421699375104","occurred_at":"2026-03-03T10:00:12Z","failed_at":"e2e","signature_id":"ecddead023c792721bff4bfa95839f57d80339b587d91d56be58becbb708a5cb","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006ce440\u003e: \n    failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n    {\n        msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n        err: \u003c*errors.StatusError | 0xc0008e2500\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n                Reason: \"NotFound\",\n                Details: {\n                    Name: \"aro-hcp-exporter\",\n                    Group: \"\",\n                    Kind: \"services\",\n                    UID: \"\",\n                    Causes: nil,\n                    RetryAfterSeconds: 0,\n                },\n                Code: 404,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006ce440\u003e: \n    failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n    {\n        msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n        err: \u003c*errors.StatusError | 0xc0008e2500\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n                Reason: \"NotFound\",\n                Details: {\n                    Name: \"aro-hcp-exporter\",\n                    Group: \"\",\n                    Kind: \"services\",\n                    UID: \"\",\n                    Causes: nil,\n                    RetryAfterSeconds: 0,\n                },\n                Code: 404,\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308","failure_pattern":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found","search_query":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028748041749532672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4075,"pr_state":"closed","pr_sha":"5b6dfb3bedbac59d4ee550297b8954f450ee93c4","final_merged_sha":"4c157031af423b1f7a6117f86f65c7962bd89ff5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T08:23:20Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"61f5a1201eb7a0bddb1c64eb1dad6c478b86182c9d6fe3268f3c2dbfa7a96232","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028748041749532672","occurred_at":"2026-03-03T08:23:20Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4293/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028742668225351680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4293,"pr_state":"closed","pr_sha":"c7d1d9e2b660a01857fa6c6049fb32f1da63c749","final_merged_sha":"c7d1d9e2b660a01857fa6c6049fb32f1da63c749","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T08:01:28Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9f363f7f5f4a3331c518181dfc49941d6f6e82d35a1828e27be33fb9f5f2a68c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4293/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028742668225351680","occurred_at":"2026-03-03T08:01:28Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4293/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028733299500781568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4293,"pr_state":"closed","pr_sha":"c7d1d9e2b660a01857fa6c6049fb32f1da63c749","final_merged_sha":"c7d1d9e2b660a01857fa6c6049fb32f1da63c749","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T07:24:45Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"09c7ab6b34b1f83b1a7c9fc1e252d9ffb202d0e7f74843d7cc2aa09b686dcd4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4293/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028733299500781568","occurred_at":"2026-03-03T07:24:45Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028666552227729408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4292,"pr_state":"closed","pr_sha":"c28b827e7f1371d1cfdc2d230c0478ccf30c5372","final_merged_sha":"c28b827e7f1371d1cfdc2d230c0478ccf30c5372","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T02:59:31Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b9d02f961fe19fdf83e640a700119a5b65b35867d42ff61315dd15f934433864","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028666552227729408","occurred_at":"2026-03-03T02:59:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028656872701038592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4290,"pr_state":"closed","pr_sha":"1710a252e4122ac0d277a20159eadd6f0506149b","final_merged_sha":"507eb199f7a4d7137d6ae5eddc82d001167895fb","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T02:21:00Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e2ab9db364836b72146e50feeb5a182c954334dabc0e87b070d9753700099f1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028656872701038592","occurred_at":"2026-03-03T02:21:00Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028656380277166080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4290,"pr_state":"closed","pr_sha":"1710a252e4122ac0d277a20159eadd6f0506149b","final_merged_sha":"507eb199f7a4d7137d6ae5eddc82d001167895fb","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T02:19:06Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3267eb6eafe36b401f9d817fb1087f6d97e605972b509dce6636b97b340edb0b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028656380277166080","occurred_at":"2026-03-03T02:19:06Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028645318098161664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4290,"pr_state":"closed","pr_sha":"1710a252e4122ac0d277a20159eadd6f0506149b","final_merged_sha":"507eb199f7a4d7137d6ae5eddc82d001167895fb","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-03T01:35:08Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fc2889a3470262acf5a312f4c453271dd5faac2e355bec27729ece162f9648b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028645318098161664","occurred_at":"2026-03-03T01:35:08Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4287/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028633136564277248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4287,"pr_state":"closed","pr_sha":"723b6e68710379451b6a670f5df9529985329888","final_merged_sha":"723b6e68710379451b6a670f5df9529985329888","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-03T00:46:44Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"24f327c4ab4f7401448f96a60c4af9455d3c9d8cf41be169009db514fec3ee0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4287/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028633136564277248","occurred_at":"2026-03-03T00:46:44Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T01:16:14.111Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T01:16:16.164Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T01:16:18.148Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=7f34c0b37c51e1001a36fba7393eddbf215791b740b755aa3e8eafb1f5011a31\ntime=2026-03-03T01:18:19.057Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4277248/providers/Microsoft.Resources/deployments/7f34c0b37c51e1001a36fba7393eddbf215791b740b755aa3e8eafb1f5011a31/operationStatuses/08584291063084145352\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4277248/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4277248/providers/Microsoft.Monitor/accounts/services-j4277248/providers/Microsoft.Authorization/roleAssignments/377c405e-8948-5cfe-b6a5-d9d917defab9\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T011640Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7ecd3fb0-d9ea-4f95-b0f0-7665a6b78546', request correlation id 'f88c8346-6f16-4ebb-a7d7-9fbd36b5743a'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4277248/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4277248/providers/Microsoft.Monitor/accounts/hcps-j4277248/providers/Microsoft.Authorization/roleAssignments/1590a3e2-478a-5448-a585-2cbc68b81369\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T011641Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7883a3ad-0349-40d7-b636-ad3d2dafb2d9', request correlation id 'f88c8346-6f16-4ebb-a7d7-9fbd36b5743a'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}